xref: /freebsd/usr.sbin/freebsd-update/freebsd-update.sh (revision 9f44a47fd07924afc035991af15d84e6585dea4f)
1#!/bin/sh
2
3#-
4# SPDX-License-Identifier: BSD-2-Clause
5#
6# Copyright 2004-2007 Colin Percival
7# All rights reserved
8#
9# Redistribution and use in source and binary forms, with or without
10# modification, are permitted providing that the following conditions
11# are met:
12# 1. Redistributions of source code must retain the above copyright
13#    notice, this list of conditions and the following disclaimer.
14# 2. Redistributions in binary form must reproduce the above copyright
15#    notice, this list of conditions and the following disclaimer in the
16#    documentation and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
22# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29
30# $FreeBSD$
31
32#### Usage function -- called from command-line handling code.
33
34# Usage instructions.  Options not listed:
35# --debug	-- don't filter output from utilities
36# --no-stats	-- don't show progress statistics while fetching files
37usage () {
38	cat <<EOF
39usage: `basename $0` [options] command ...
40
41Options:
42  -b basedir   -- Operate on a system mounted at basedir
43                  (default: /)
44  -d workdir   -- Store working files in workdir
45                  (default: /var/db/freebsd-update/)
46  -f conffile  -- Read configuration options from conffile
47                  (default: /etc/freebsd-update.conf)
48  -F           -- Force a fetch operation to proceed in the
49                  case of an unfinished upgrade
50  -j jail      -- Operate on the given jail specified by jid or name
51  -k KEY       -- Trust an RSA key with SHA256 hash of KEY
52  -r release   -- Target for upgrade (e.g., 13.2-RELEASE)
53  -s server    -- Server from which to fetch updates
54                  (default: update.FreeBSD.org)
55  -t address   -- Mail output of cron command, if any, to address
56                  (default: root)
57  --not-running-from-cron
58               -- Run without a tty, for use by automated tools
59  --currently-running release
60               -- Update as if currently running this release
61Commands:
62  fetch        -- Fetch updates from server
63  cron         -- Sleep rand(3600) seconds, fetch updates, and send an
64                  email if updates were found
65  upgrade      -- Fetch upgrades to FreeBSD version specified via -r option
66  updatesready -- Check if there are fetched updates ready to install
67  install      -- Install downloaded updates or upgrades
68  rollback     -- Uninstall most recently installed updates
69  IDS          -- Compare the system against an index of "known good" files
70  showconfig   -- Show configuration
71EOF
72	exit 0
73}
74
75#### Configuration processing functions
76
77#-
78# Configuration options are set in the following order of priority:
79# 1. Command line options
80# 2. Configuration file options
81# 3. Default options
82# In addition, certain options (e.g., IgnorePaths) can be specified multiple
83# times and (as long as these are all in the same place, e.g., inside the
84# configuration file) they will accumulate.  Finally, because the path to the
85# configuration file can be specified at the command line, the entire command
86# line must be processed before we start reading the configuration file.
87#
88# Sound like a mess?  It is.  Here's how we handle this:
89# 1. Initialize CONFFILE and all the options to "".
90# 2. Process the command line.  Throw an error if a non-accumulating option
91#    is specified twice.
92# 3. If CONFFILE is "", set CONFFILE to /etc/freebsd-update.conf .
93# 4. For all the configuration options X, set X_saved to X.
94# 5. Initialize all the options to "".
95# 6. Read CONFFILE line by line, parsing options.
96# 7. For each configuration option X, set X to X_saved iff X_saved is not "".
97# 8. Repeat steps 4-7, except setting options to their default values at (6).
98
99CONFIGOPTIONS="KEYPRINT WORKDIR SERVERNAME MAILTO ALLOWADD ALLOWDELETE
100    KEEPMODIFIEDMETADATA COMPONENTS IGNOREPATHS UPDATEIFUNMODIFIED
101    BASEDIR VERBOSELEVEL TARGETRELEASE STRICTCOMPONENTS MERGECHANGES
102    IDSIGNOREPATHS BACKUPKERNEL BACKUPKERNELDIR BACKUPKERNELSYMBOLFILES"
103
104# Set all the configuration options to "".
105nullconfig () {
106	for X in ${CONFIGOPTIONS}; do
107		eval ${X}=""
108	done
109}
110
111# For each configuration option X, set X_saved to X.
112saveconfig () {
113	for X in ${CONFIGOPTIONS}; do
114		eval ${X}_saved=\$${X}
115	done
116}
117
118# For each configuration option X, set X to X_saved if X_saved is not "".
119mergeconfig () {
120	for X in ${CONFIGOPTIONS}; do
121		eval _=\$${X}_saved
122		if ! [ -z "${_}" ]; then
123			eval ${X}=\$${X}_saved
124		fi
125	done
126}
127
128# Set the trusted keyprint.
129config_KeyPrint () {
130	if [ -z ${KEYPRINT} ]; then
131		KEYPRINT=$1
132	else
133		return 1
134	fi
135}
136
137# Set the working directory.
138config_WorkDir () {
139	if [ -z ${WORKDIR} ]; then
140		WORKDIR=$1
141	else
142		return 1
143	fi
144}
145
146# Set the name of the server (pool) from which to fetch updates
147config_ServerName () {
148	if [ -z ${SERVERNAME} ]; then
149		SERVERNAME=$1
150	else
151		return 1
152	fi
153}
154
155# Set the address to which 'cron' output will be mailed.
156config_MailTo () {
157	if [ -z ${MAILTO} ]; then
158		MAILTO=$1
159	else
160		return 1
161	fi
162}
163
164# Set whether FreeBSD Update is allowed to add files (or directories, or
165# symlinks) which did not previously exist.
166config_AllowAdd () {
167	if [ -z ${ALLOWADD} ]; then
168		case $1 in
169		[Yy][Ee][Ss])
170			ALLOWADD=yes
171			;;
172		[Nn][Oo])
173			ALLOWADD=no
174			;;
175		*)
176			return 1
177			;;
178		esac
179	else
180		return 1
181	fi
182}
183
184# Set whether FreeBSD Update is allowed to remove files/directories/symlinks.
185config_AllowDelete () {
186	if [ -z ${ALLOWDELETE} ]; then
187		case $1 in
188		[Yy][Ee][Ss])
189			ALLOWDELETE=yes
190			;;
191		[Nn][Oo])
192			ALLOWDELETE=no
193			;;
194		*)
195			return 1
196			;;
197		esac
198	else
199		return 1
200	fi
201}
202
203# Set whether FreeBSD Update should keep existing inode ownership,
204# permissions, and flags, in the event that they have been modified locally
205# after the release.
206config_KeepModifiedMetadata () {
207	if [ -z ${KEEPMODIFIEDMETADATA} ]; then
208		case $1 in
209		[Yy][Ee][Ss])
210			KEEPMODIFIEDMETADATA=yes
211			;;
212		[Nn][Oo])
213			KEEPMODIFIEDMETADATA=no
214			;;
215		*)
216			return 1
217			;;
218		esac
219	else
220		return 1
221	fi
222}
223
224# Add to the list of components which should be kept updated.
225config_Components () {
226	for C in $@; do
227		COMPONENTS="${COMPONENTS} ${C}"
228	done
229}
230
231# Remove src component from list if it isn't installed
232finalize_components_config () {
233	COMPONENTS=""
234	for C in $@; do
235		if [ "$C" = "src" ]; then
236			if [ -e "${BASEDIR}/usr/src/COPYRIGHT" ]; then
237				COMPONENTS="${COMPONENTS} ${C}"
238			else
239				echo "src component not installed, skipped"
240			fi
241		else
242			COMPONENTS="${COMPONENTS} ${C}"
243		fi
244	done
245}
246
247# Add to the list of paths under which updates will be ignored.
248config_IgnorePaths () {
249	for C in $@; do
250		IGNOREPATHS="${IGNOREPATHS} ${C}"
251	done
252}
253
254# Add to the list of paths which IDS should ignore.
255config_IDSIgnorePaths () {
256	for C in $@; do
257		IDSIGNOREPATHS="${IDSIGNOREPATHS} ${C}"
258	done
259}
260
261# Add to the list of paths within which updates will be performed only if the
262# file on disk has not been modified locally.
263config_UpdateIfUnmodified () {
264	for C in $@; do
265		UPDATEIFUNMODIFIED="${UPDATEIFUNMODIFIED} ${C}"
266	done
267}
268
269# Add to the list of paths within which updates to text files will be merged
270# instead of overwritten.
271config_MergeChanges () {
272	for C in $@; do
273		MERGECHANGES="${MERGECHANGES} ${C}"
274	done
275}
276
277# Work on a FreeBSD installation mounted under $1
278config_BaseDir () {
279	if [ -z ${BASEDIR} ]; then
280		BASEDIR=$1
281	else
282		return 1
283	fi
284}
285
286# When fetching upgrades, should we assume the user wants exactly the
287# components listed in COMPONENTS, rather than trying to guess based on
288# what's currently installed?
289config_StrictComponents () {
290	if [ -z ${STRICTCOMPONENTS} ]; then
291		case $1 in
292		[Yy][Ee][Ss])
293			STRICTCOMPONENTS=yes
294			;;
295		[Nn][Oo])
296			STRICTCOMPONENTS=no
297			;;
298		*)
299			return 1
300			;;
301		esac
302	else
303		return 1
304	fi
305}
306
307# Upgrade to FreeBSD $1
308config_TargetRelease () {
309	if [ -z ${TARGETRELEASE} ]; then
310		TARGETRELEASE=$1
311	else
312		return 1
313	fi
314	if echo ${TARGETRELEASE} | grep -qE '^[0-9.]+$'; then
315		TARGETRELEASE="${TARGETRELEASE}-RELEASE"
316	fi
317}
318
319# Pretend current release is FreeBSD $1
320config_SourceRelease () {
321	UNAME_r=$1
322	if echo ${UNAME_r} | grep -qE '^[0-9.]+$'; then
323		UNAME_r="${UNAME_r}-RELEASE"
324	fi
325	export UNAME_r
326}
327
328# Get the Jail's path and the version of its installed userland
329config_TargetJail () {
330	JAIL=$1
331	UNAME_r=$(freebsd-version -j ${JAIL})
332	BASEDIR=$(jls -j ${JAIL} -h path | awk 'NR == 2 {print}')
333	if [ -z ${BASEDIR} ] || [ -z ${UNAME_r} ]; then
334		echo "The specified jail either doesn't exist or" \
335		      "does not have freebsd-version."
336		exit 1
337	fi
338	export UNAME_r
339}
340
341# Define what happens to output of utilities
342config_VerboseLevel () {
343	if [ -z ${VERBOSELEVEL} ]; then
344		case $1 in
345		[Dd][Ee][Bb][Uu][Gg])
346			VERBOSELEVEL=debug
347			;;
348		[Nn][Oo][Ss][Tt][Aa][Tt][Ss])
349			VERBOSELEVEL=nostats
350			;;
351		[Ss][Tt][Aa][Tt][Ss])
352			VERBOSELEVEL=stats
353			;;
354		*)
355			return 1
356			;;
357		esac
358	else
359		return 1
360	fi
361}
362
363config_BackupKernel () {
364	if [ -z ${BACKUPKERNEL} ]; then
365		case $1 in
366		[Yy][Ee][Ss])
367			BACKUPKERNEL=yes
368			;;
369		[Nn][Oo])
370			BACKUPKERNEL=no
371			;;
372		*)
373			return 1
374			;;
375		esac
376	else
377		return 1
378	fi
379}
380
381config_BackupKernelDir () {
382	if [ -z ${BACKUPKERNELDIR} ]; then
383		if [ -z "$1" ]; then
384			echo "BackupKernelDir set to empty dir"
385			return 1
386		fi
387
388		# We check for some paths which would be extremely odd
389		# to use, but which could cause a lot of problems if
390		# used.
391		case $1 in
392		/|/bin|/boot|/etc|/lib|/libexec|/sbin|/usr|/var)
393			echo "BackupKernelDir set to invalid path $1"
394			return 1
395			;;
396		/*)
397			BACKUPKERNELDIR=$1
398			;;
399		*)
400			echo "BackupKernelDir ($1) is not an absolute path"
401			return 1
402			;;
403		esac
404	else
405		return 1
406	fi
407}
408
409config_BackupKernelSymbolFiles () {
410	if [ -z ${BACKUPKERNELSYMBOLFILES} ]; then
411		case $1 in
412		[Yy][Ee][Ss])
413			BACKUPKERNELSYMBOLFILES=yes
414			;;
415		[Nn][Oo])
416			BACKUPKERNELSYMBOLFILES=no
417			;;
418		*)
419			return 1
420			;;
421		esac
422	else
423		return 1
424	fi
425}
426
427config_CreateBootEnv () {
428	if [ -z ${BOOTENV} ]; then
429		case $1 in
430		[Yy][Ee][Ss])
431			BOOTENV=yes
432			;;
433		[Nn][Oo])
434			BOOTENV=no
435			;;
436		*)
437			return 1
438			;;
439		esac
440	else
441		return 1
442	fi
443}
444# Handle one line of configuration
445configline () {
446	if [ $# -eq 0 ]; then
447		return
448	fi
449
450	OPT=$1
451	shift
452	config_${OPT} $@
453}
454
455#### Parameter handling functions.
456
457# Initialize parameters to null, just in case they're
458# set in the environment.
459init_params () {
460	# Configration settings
461	nullconfig
462
463	# No configuration file set yet
464	CONFFILE=""
465
466	# No commands specified yet
467	COMMANDS=""
468
469	# Force fetch to proceed
470	FORCEFETCH=0
471
472	# Run without a TTY
473	NOTTYOK=0
474
475	# Fetched first in a chain of commands
476	ISFETCHED=0
477}
478
479# Parse the command line
480parse_cmdline () {
481	while [ $# -gt 0 ]; do
482		case "$1" in
483		# Location of configuration file
484		-f)
485			if [ $# -eq 1 ]; then usage; fi
486			if [ ! -z "${CONFFILE}" ]; then usage; fi
487			shift; CONFFILE="$1"
488			;;
489		-F)
490			FORCEFETCH=1
491			;;
492		--not-running-from-cron)
493			NOTTYOK=1
494			;;
495		--currently-running)
496			shift
497			config_SourceRelease $1 || usage
498			;;
499
500		# Configuration file equivalents
501		-b)
502			if [ $# -eq 1 ]; then usage; fi; shift
503			config_BaseDir $1 || usage
504			;;
505		-d)
506			if [ $# -eq 1 ]; then usage; fi; shift
507			config_WorkDir $1 || usage
508			;;
509		-j)
510			if [ $# -eq 1 ]; then usage; fi; shift
511			config_TargetJail $1 || usage
512			;;
513		-k)
514			if [ $# -eq 1 ]; then usage; fi; shift
515			config_KeyPrint $1 || usage
516			;;
517		-s)
518			if [ $# -eq 1 ]; then usage; fi; shift
519			config_ServerName $1 || usage
520			;;
521		-r)
522			if [ $# -eq 1 ]; then usage; fi; shift
523			config_TargetRelease $1 || usage
524			;;
525		-t)
526			if [ $# -eq 1 ]; then usage; fi; shift
527			config_MailTo $1 || usage
528			;;
529		-v)
530			if [ $# -eq 1 ]; then usage; fi; shift
531			config_VerboseLevel $1 || usage
532			;;
533
534		# Aliases for "-v debug" and "-v nostats"
535		--debug)
536			config_VerboseLevel debug || usage
537			;;
538		--no-stats)
539			config_VerboseLevel nostats || usage
540			;;
541
542		# Commands
543		cron | fetch | upgrade | updatesready | install | rollback |\
544		IDS | showconfig)
545			COMMANDS="${COMMANDS} $1"
546			;;
547
548		# Anything else is an error
549		*)
550			usage
551			;;
552		esac
553		shift
554	done
555
556	# Make sure we have at least one command
557	if [ -z "${COMMANDS}" ]; then
558		usage
559	fi
560}
561
562# Parse the configuration file
563parse_conffile () {
564	# If a configuration file was specified on the command line, check
565	# that it exists and is readable.
566	if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then
567		echo -n "File does not exist "
568		echo -n "or is not readable: "
569		echo ${CONFFILE}
570		exit 1
571	fi
572
573	# If a configuration file was not specified on the command line,
574	# use the default configuration file path.  If that default does
575	# not exist, give up looking for any configuration.
576	if [ -z "${CONFFILE}" ]; then
577		CONFFILE="/etc/freebsd-update.conf"
578		if [ ! -r "${CONFFILE}" ]; then
579			return
580		fi
581	fi
582
583	# Save the configuration options specified on the command line, and
584	# clear all the options in preparation for reading the config file.
585	saveconfig
586	nullconfig
587
588	# Read the configuration file.  Anything after the first '#' is
589	# ignored, and any blank lines are ignored.
590	L=0
591	while read LINE; do
592		L=$(($L + 1))
593		LINEX=`echo "${LINE}" | cut -f 1 -d '#'`
594		if ! configline ${LINEX}; then
595			echo "Error processing configuration file, line $L:"
596			echo "==> ${LINE}"
597			exit 1
598		fi
599	done < ${CONFFILE}
600
601	# Merge the settings read from the configuration file with those
602	# provided at the command line.
603	mergeconfig
604}
605
606# Provide some default parameters
607default_params () {
608	# Save any parameters already configured, and clear the slate
609	saveconfig
610	nullconfig
611
612	# Default configurations
613	config_WorkDir /var/db/freebsd-update
614	config_MailTo root
615	config_AllowAdd yes
616	config_AllowDelete yes
617	config_KeepModifiedMetadata yes
618	config_BaseDir /
619	config_VerboseLevel stats
620	config_StrictComponents no
621	config_BackupKernel yes
622	config_BackupKernelDir /boot/kernel.old
623	config_BackupKernelSymbolFiles no
624	config_CreateBootEnv yes
625
626	# Merge these defaults into the earlier-configured settings
627	mergeconfig
628}
629
630# Set utility output filtering options, based on ${VERBOSELEVEL}
631fetch_setup_verboselevel () {
632	case ${VERBOSELEVEL} in
633	debug)
634		QUIETREDIR="/dev/stderr"
635		QUIETFLAG=" "
636		STATSREDIR="/dev/stderr"
637		DDSTATS=".."
638		XARGST="-t"
639		NDEBUG=" "
640		;;
641	nostats)
642		QUIETREDIR=""
643		QUIETFLAG=""
644		STATSREDIR="/dev/null"
645		DDSTATS=".."
646		XARGST=""
647		NDEBUG=""
648		;;
649	stats)
650		QUIETREDIR="/dev/null"
651		QUIETFLAG="-q"
652		STATSREDIR="/dev/stdout"
653		DDSTATS=""
654		XARGST=""
655		NDEBUG="-n"
656		;;
657	esac
658}
659
660# Perform sanity checks and set some final parameters
661# in preparation for fetching files.  Figure out which
662# set of updates should be downloaded: If the user is
663# running *-p[0-9]+, strip off the last part; if the
664# user is running -SECURITY, call it -RELEASE.  Chdir
665# into the working directory.
666fetchupgrade_check_params () {
667	export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"
668
669	_SERVERNAME_z=\
670"SERVERNAME must be given via command line or configuration file."
671	_KEYPRINT_z="Key must be given via -k option or configuration file."
672	_KEYPRINT_bad="Invalid key fingerprint: "
673	_WORKDIR_bad="Directory does not exist or is not writable: "
674	_WORKDIR_bad2="Directory is not on a persistent filesystem: "
675
676	if [ -z "${SERVERNAME}" ]; then
677		echo -n "`basename $0`: "
678		echo "${_SERVERNAME_z}"
679		exit 1
680	fi
681	if [ -z "${KEYPRINT}" ]; then
682		echo -n "`basename $0`: "
683		echo "${_KEYPRINT_z}"
684		exit 1
685	fi
686	if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
687		echo -n "`basename $0`: "
688		echo -n "${_KEYPRINT_bad}"
689		echo ${KEYPRINT}
690		exit 1
691	fi
692	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
693		echo -n "`basename $0`: "
694		echo -n "${_WORKDIR_bad}"
695		echo ${WORKDIR}
696		exit 1
697	fi
698	case `df -T ${WORKDIR}` in */dev/md[0-9]* | *tmpfs*)
699		echo -n "`basename $0`: "
700		echo -n "${_WORKDIR_bad2}"
701		echo ${WORKDIR}
702		exit 1
703		;;
704	esac
705	chmod 700 ${WORKDIR}
706	cd ${WORKDIR} || exit 1
707
708	# Generate release number.  The s/SECURITY/RELEASE/ bit exists
709	# to provide an upgrade path for FreeBSD Update 1.x users, since
710	# the kernels provided by FreeBSD Update 1.x are always labelled
711	# as X.Y-SECURITY.
712	RELNUM=`uname -r |
713	    sed -E 's,-p[0-9]+,,' |
714	    sed -E 's,-SECURITY,-RELEASE,'`
715	ARCH=`uname -m`
716	FETCHDIR=${RELNUM}/${ARCH}
717	PATCHDIR=${RELNUM}/${ARCH}/bp
718
719	# Disallow upgrade from a version that is not a release
720	case ${RELNUM} in
721	*-RELEASE | *-ALPHA*  | *-BETA* | *-RC*)
722		;;
723	*)
724		echo -n "`basename $0`: "
725		cat <<- EOF
726			Cannot upgrade from a version that is not a release
727			(including alpha, beta and release candidates)
728			using `basename $0`. Instead, FreeBSD can be directly
729			upgraded by source or upgraded to a RELEASE/RELENG version
730			prior to running `basename $0`.
731			Currently running: ${RELNUM}
732		EOF
733		exit 1
734		;;
735	esac
736
737	# Figure out what directory contains the running kernel
738	BOOTFILE=`sysctl -n kern.bootfile`
739	KERNELDIR=${BOOTFILE%/kernel}
740	if ! [ -d ${KERNELDIR} ]; then
741		echo "Cannot identify running kernel"
742		exit 1
743	fi
744
745	# Figure out what kernel configuration is running.  We start with
746	# the output of `uname -i`, and then make the following adjustments:
747	# 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
748	# file says "ident SMP-GENERIC", I don't know...
749	# 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
750	# _and_ `sysctl kern.version` contains a line which ends "/SMP", then
751	# we're running an SMP kernel.  This mis-identification is a bug
752	# which was fixed in 6.2-STABLE.
753	KERNCONF=`uname -i`
754	if [ ${KERNCONF} = "SMP-GENERIC" ]; then
755		KERNCONF=SMP
756	fi
757	if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
758		if sysctl kern.version | grep -qE '/SMP$'; then
759			KERNCONF=SMP
760		fi
761	fi
762
763	# Define some paths
764	BSPATCH=/usr/bin/bspatch
765	SHA256=/sbin/sha256
766	PHTTPGET=/usr/libexec/phttpget
767
768	# Set up variables relating to VERBOSELEVEL
769	fetch_setup_verboselevel
770
771	# Construct a unique name from ${BASEDIR}
772	BDHASH=`echo ${BASEDIR} | sha256 -q`
773}
774
775# Perform sanity checks etc. before fetching updates.
776fetch_check_params () {
777	fetchupgrade_check_params
778
779	if ! [ -z "${TARGETRELEASE}" ]; then
780		echo -n "`basename $0`: "
781		echo -n "-r option is meaningless with 'fetch' command.  "
782		echo "(Did you mean 'upgrade' instead?)"
783		exit 1
784	fi
785
786	# Check that we have updates ready to install
787	if [ -f ${BDHASH}-install/kerneldone -a $FORCEFETCH -eq 0 ]; then
788		echo "You have a partially completed upgrade pending"
789		echo "Run '$0 install' first."
790		echo "Run '$0 fetch -F' to proceed anyway."
791		exit 1
792	fi
793}
794
795# Perform sanity checks etc. before fetching upgrades.
796upgrade_check_params () {
797	fetchupgrade_check_params
798
799	# Unless set otherwise, we're upgrading to the same kernel config.
800	NKERNCONF=${KERNCONF}
801
802	# We need TARGETRELEASE set
803	_TARGETRELEASE_z="Release target must be specified via -r option."
804	if [ -z "${TARGETRELEASE}" ]; then
805		echo -n "`basename $0`: "
806		echo "${_TARGETRELEASE_z}"
807		exit 1
808	fi
809
810	# The target release should be != the current release.
811	if [ "${TARGETRELEASE}" = "${RELNUM}" ]; then
812		echo -n "`basename $0`: "
813		echo "Cannot upgrade from ${RELNUM} to itself"
814		exit 1
815	fi
816
817	# Turning off AllowAdd or AllowDelete is a bad idea for upgrades.
818	if [ "${ALLOWADD}" = "no" ]; then
819		echo -n "`basename $0`: "
820		echo -n "WARNING: \"AllowAdd no\" is a bad idea "
821		echo "when upgrading between releases."
822		echo
823	fi
824	if [ "${ALLOWDELETE}" = "no" ]; then
825		echo -n "`basename $0`: "
826		echo -n "WARNING: \"AllowDelete no\" is a bad idea "
827		echo "when upgrading between releases."
828		echo
829	fi
830
831	# Set EDITOR to /usr/bin/vi if it isn't already set
832	: ${EDITOR:='/usr/bin/vi'}
833}
834
835# Perform sanity checks and set some final parameters in
836# preparation for installing updates.
837install_check_params () {
838	# Check that we are root.  All sorts of things won't work otherwise.
839	if [ `id -u` != 0 ]; then
840		echo "You must be root to run this."
841		exit 1
842	fi
843
844	# Check that securelevel <= 0.  Otherwise we can't update schg files.
845	if [ `sysctl -n kern.securelevel` -gt 0 ]; then
846		echo "Updates cannot be installed when the system securelevel"
847		echo "is greater than zero."
848		exit 1
849	fi
850
851	# Check that we have a working directory
852	_WORKDIR_bad="Directory does not exist or is not writable: "
853	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
854		echo -n "`basename $0`: "
855		echo -n "${_WORKDIR_bad}"
856		echo ${WORKDIR}
857		exit 1
858	fi
859	cd ${WORKDIR} || exit 1
860
861	# Construct a unique name from ${BASEDIR}
862	BDHASH=`echo ${BASEDIR} | sha256 -q`
863
864	# Check that we have updates ready to install
865	if ! [ -L ${BDHASH}-install ]; then
866		echo "No updates are available to install."
867		if [ $ISFETCHED -eq 0 ]; then
868			echo "Run '$0 fetch' first."
869			exit 2
870		fi
871		exit 0
872	fi
873	if ! [ -f ${BDHASH}-install/INDEX-OLD ] ||
874	    ! [ -f ${BDHASH}-install/INDEX-NEW ]; then
875		echo "Update manifest is corrupt -- this should never happen."
876		echo "Re-run '$0 fetch'."
877		exit 1
878	fi
879
880	# Figure out what directory contains the running kernel
881	BOOTFILE=`sysctl -n kern.bootfile`
882	KERNELDIR=${BOOTFILE%/kernel}
883	if ! [ -d ${KERNELDIR} ]; then
884		echo "Cannot identify running kernel"
885		exit 1
886	fi
887}
888
889# Creates a new boot environment
890install_create_be () {
891	# Figure out if we're running in a jail and return if we are
892	if [ `sysctl -n security.jail.jailed` = 1 ]; then
893		return 1
894	fi
895	# Operating on roots that aren't located at / will, more often than not,
896	# not touch the boot environment.
897	if [ "$BASEDIR" != "/" ]; then
898		return 1
899	fi
900	# Create a boot environment if enabled
901	if [ ${BOOTENV} = yes ]; then
902		bectl check 2>/dev/null
903		case $? in
904			0)
905				# Boot environment are supported
906				CREATEBE=yes
907				;;
908			255)
909				# Boot environments are not supported
910				CREATEBE=no
911				;;
912			*)
913				# If bectl returns an unexpected exit code, don't create a BE
914				CREATEBE=no
915				;;
916		esac
917		if [ ${CREATEBE} = yes ]; then
918			echo -n "Creating snapshot of existing boot environment... "
919			VERSION=`freebsd-version -ku | sort -V | tail -n 1`
920			TIMESTAMP=`date +"%Y-%m-%d_%H%M%S"`
921			bectl create ${VERSION}_${TIMESTAMP}
922			if [ $? -eq 0 ]; then
923				echo "done.";
924			else
925				echo "failed."
926				exit 1
927			fi
928		fi
929	fi
930}
931
932# Perform sanity checks and set some final parameters in
933# preparation for UNinstalling updates.
934rollback_check_params () {
935	# Check that we are root.  All sorts of things won't work otherwise.
936	if [ `id -u` != 0 ]; then
937		echo "You must be root to run this."
938		exit 1
939	fi
940
941	# Check that we have a working directory
942	_WORKDIR_bad="Directory does not exist or is not writable: "
943	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
944		echo -n "`basename $0`: "
945		echo -n "${_WORKDIR_bad}"
946		echo ${WORKDIR}
947		exit 1
948	fi
949	cd ${WORKDIR} || exit 1
950
951	# Construct a unique name from ${BASEDIR}
952	BDHASH=`echo ${BASEDIR} | sha256 -q`
953
954	# Check that we have updates ready to rollback
955	if ! [ -L ${BDHASH}-rollback ]; then
956		echo "No rollback directory found."
957		exit 1
958	fi
959	if ! [ -f ${BDHASH}-rollback/INDEX-OLD ] ||
960	    ! [ -f ${BDHASH}-rollback/INDEX-NEW ]; then
961		echo "Update manifest is corrupt -- this should never happen."
962		exit 1
963	fi
964}
965
966# Perform sanity checks and set some final parameters
967# in preparation for comparing the system against the
968# published index.  Figure out which index we should
969# compare against: If the user is running *-p[0-9]+,
970# strip off the last part; if the user is running
971# -SECURITY, call it -RELEASE.  Chdir into the working
972# directory.
973IDS_check_params () {
974	export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"
975
976	_SERVERNAME_z=\
977"SERVERNAME must be given via command line or configuration file."
978	_KEYPRINT_z="Key must be given via -k option or configuration file."
979	_KEYPRINT_bad="Invalid key fingerprint: "
980	_WORKDIR_bad="Directory does not exist or is not writable: "
981
982	if [ -z "${SERVERNAME}" ]; then
983		echo -n "`basename $0`: "
984		echo "${_SERVERNAME_z}"
985		exit 1
986	fi
987	if [ -z "${KEYPRINT}" ]; then
988		echo -n "`basename $0`: "
989		echo "${_KEYPRINT_z}"
990		exit 1
991	fi
992	if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
993		echo -n "`basename $0`: "
994		echo -n "${_KEYPRINT_bad}"
995		echo ${KEYPRINT}
996		exit 1
997	fi
998	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
999		echo -n "`basename $0`: "
1000		echo -n "${_WORKDIR_bad}"
1001		echo ${WORKDIR}
1002		exit 1
1003	fi
1004	cd ${WORKDIR} || exit 1
1005
1006	# Generate release number.  The s/SECURITY/RELEASE/ bit exists
1007	# to provide an upgrade path for FreeBSD Update 1.x users, since
1008	# the kernels provided by FreeBSD Update 1.x are always labelled
1009	# as X.Y-SECURITY.
1010	RELNUM=`uname -r |
1011	    sed -E 's,-p[0-9]+,,' |
1012	    sed -E 's,-SECURITY,-RELEASE,'`
1013	ARCH=`uname -m`
1014	FETCHDIR=${RELNUM}/${ARCH}
1015	PATCHDIR=${RELNUM}/${ARCH}/bp
1016
1017	# Figure out what directory contains the running kernel
1018	BOOTFILE=`sysctl -n kern.bootfile`
1019	KERNELDIR=${BOOTFILE%/kernel}
1020	if ! [ -d ${KERNELDIR} ]; then
1021		echo "Cannot identify running kernel"
1022		exit 1
1023	fi
1024
1025	# Figure out what kernel configuration is running.  We start with
1026	# the output of `uname -i`, and then make the following adjustments:
1027	# 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
1028	# file says "ident SMP-GENERIC", I don't know...
1029	# 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
1030	# _and_ `sysctl kern.version` contains a line which ends "/SMP", then
1031	# we're running an SMP kernel.  This mis-identification is a bug
1032	# which was fixed in 6.2-STABLE.
1033	KERNCONF=`uname -i`
1034	if [ ${KERNCONF} = "SMP-GENERIC" ]; then
1035		KERNCONF=SMP
1036	fi
1037	if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
1038		if sysctl kern.version | grep -qE '/SMP$'; then
1039			KERNCONF=SMP
1040		fi
1041	fi
1042
1043	# Define some paths
1044	SHA256=/sbin/sha256
1045	PHTTPGET=/usr/libexec/phttpget
1046
1047	# Set up variables relating to VERBOSELEVEL
1048	fetch_setup_verboselevel
1049}
1050
1051#### Core functionality -- the actual work gets done here
1052
1053# Use an SRV query to pick a server.  If the SRV query doesn't provide
1054# a useful answer, use the server name specified by the user.
1055# Put another way... look up _http._tcp.${SERVERNAME} and pick a server
1056# from that; or if no servers are returned, use ${SERVERNAME}.
1057# This allows a user to specify "portsnap.freebsd.org" (in which case
1058# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org"
1059# (in which case portsnap will use that particular server, since there
1060# won't be an SRV entry for that name).
1061#
1062# We ignore the Port field, since we are always going to use port 80.
1063
1064# Fetch the mirror list, but do not pick a mirror yet.  Returns 1 if
1065# no mirrors are available for any reason.
1066fetch_pick_server_init () {
1067	: > serverlist_tried
1068
1069# Check that host(1) exists (i.e., that the system wasn't built with the
1070# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist.
1071	if ! which -s host; then
1072		: > serverlist_full
1073		return 1
1074	fi
1075
1076	echo -n "Looking up ${SERVERNAME} mirrors... "
1077
1078# Issue the SRV query and pull out the Priority, Weight, and Target fields.
1079# BIND 9 prints "$name has SRV record ..." while BIND 8 prints
1080# "$name server selection ..."; we allow either format.
1081	MLIST="_http._tcp.${SERVERNAME}"
1082	host -t srv "${MLIST}" |
1083	    sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" |
1084	    cut -f 1,2,4 -d ' ' |
1085	    sed -e 's/\.$//' |
1086	    sort > serverlist_full
1087
1088# If no records, give up -- we'll just use the server name we were given.
1089	if [ `wc -l < serverlist_full` -eq 0 ]; then
1090		echo "none found."
1091		return 1
1092	fi
1093
1094# Report how many mirrors we found.
1095	echo `wc -l < serverlist_full` "mirrors found."
1096
1097# Generate a random seed for use in picking mirrors.  If HTTP_PROXY
1098# is set, this will be used to generate the seed; otherwise, the seed
1099# will be random.
1100	if [ -n "${HTTP_PROXY}${http_proxy}" ]; then
1101		RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" |
1102		    tr -d 'a-f' |
1103		    cut -c 1-9`
1104	else
1105		RANDVALUE=`jot -r 1 0 999999999`
1106	fi
1107}
1108
1109# Pick a mirror.  Returns 1 if we have run out of mirrors to try.
1110fetch_pick_server () {
1111# Generate a list of not-yet-tried mirrors
1112	sort serverlist_tried |
1113	    comm -23 serverlist_full - > serverlist
1114
1115# Have we run out of mirrors?
1116	if [ `wc -l < serverlist` -eq 0 ]; then
1117		cat <<- EOF
1118			No mirrors remaining, giving up.
1119
1120			This may be because upgrading from this platform (${ARCH})
1121			or release (${RELNUM}) is unsupported by `basename $0`. Only
1122			platforms with Tier 1 support can be upgraded by `basename $0`.
1123			See https://www.freebsd.org/platforms/ for more info.
1124
1125			If unsupported, FreeBSD must be upgraded by source.
1126		EOF
1127		return 1
1128	fi
1129
1130# Find the highest priority level (lowest numeric value).
1131	SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1`
1132
1133# Add up the weights of the response lines at that priority level.
1134	SRV_WSUM=0;
1135	while read X; do
1136		case "$X" in
1137		${SRV_PRIORITY}\ *)
1138			SRV_W=`echo $X | cut -f 2 -d ' '`
1139			SRV_WSUM=$(($SRV_WSUM + $SRV_W))
1140			;;
1141		esac
1142	done < serverlist
1143
1144# If all the weights are 0, pretend that they are all 1 instead.
1145	if [ ${SRV_WSUM} -eq 0 ]; then
1146		SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l`
1147		SRV_W_ADD=1
1148	else
1149		SRV_W_ADD=0
1150	fi
1151
1152# Pick a value between 0 and the sum of the weights - 1
1153	SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}`
1154
1155# Read through the list of mirrors and set SERVERNAME.  Write the line
1156# corresponding to the mirror we selected into serverlist_tried so that
1157# we won't try it again.
1158	while read X; do
1159		case "$X" in
1160		${SRV_PRIORITY}\ *)
1161			SRV_W=`echo $X | cut -f 2 -d ' '`
1162			SRV_W=$(($SRV_W + $SRV_W_ADD))
1163			if [ $SRV_RND -lt $SRV_W ]; then
1164				SERVERNAME=`echo $X | cut -f 3 -d ' '`
1165				echo "$X" >> serverlist_tried
1166				break
1167			else
1168				SRV_RND=$(($SRV_RND - $SRV_W))
1169			fi
1170			;;
1171		esac
1172	done < serverlist
1173}
1174
1175# Take a list of ${oldhash}|${newhash} and output a list of needed patches,
1176# i.e., those for which we have ${oldhash} and don't have ${newhash}.
1177fetch_make_patchlist () {
1178	grep -vE "^([0-9a-f]{64})\|\1$" |
1179	    tr '|' ' ' |
1180		while read X Y; do
1181			if [ -f "files/${Y}.gz" ] ||
1182			    [ ! -f "files/${X}.gz" ]; then
1183				continue
1184			fi
1185			echo "${X}|${Y}"
1186		done | sort -u
1187}
1188
1189# Print user-friendly progress statistics
1190fetch_progress () {
1191	LNC=0
1192	while read x; do
1193		LNC=$(($LNC + 1))
1194		if [ $(($LNC % 10)) = 0 ]; then
1195			echo -n $LNC
1196		elif [ $(($LNC % 2)) = 0 ]; then
1197			echo -n .
1198		fi
1199	done
1200	echo -n " "
1201}
1202
1203# Function for asking the user if everything is ok
1204continuep () {
1205	while read -p "Does this look reasonable (y/n)? " CONTINUE; do
1206		case "${CONTINUE}" in
1207		y*)
1208			return 0
1209			;;
1210		n*)
1211			return 1
1212			;;
1213		esac
1214	done
1215}
1216
1217# Initialize the working directory
1218workdir_init () {
1219	mkdir -p files
1220	touch tINDEX.present
1221}
1222
1223# Check that we have a public key with an appropriate hash, or
1224# fetch the key if it doesn't exist.  Returns 1 if the key has
1225# not yet been fetched.
1226fetch_key () {
1227	if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
1228		return 0
1229	fi
1230
1231	echo -n "Fetching public key from ${SERVERNAME}... "
1232	rm -f pub.ssl
1233	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/pub.ssl \
1234	    2>${QUIETREDIR} || true
1235	if ! [ -r pub.ssl ]; then
1236		echo "failed."
1237		return 1
1238	fi
1239	if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
1240		echo "key has incorrect hash."
1241		rm -f pub.ssl
1242		return 1
1243	fi
1244	echo "done."
1245}
1246
1247# Fetch metadata signature, aka "tag".
1248fetch_tag () {
1249	echo -n "Fetching metadata signature "
1250	echo ${NDEBUG} "for ${RELNUM} from ${SERVERNAME}... "
1251	rm -f latest.ssl
1252	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/latest.ssl	\
1253	    2>${QUIETREDIR} || true
1254	if ! [ -r latest.ssl ]; then
1255		echo "failed."
1256		return 1
1257	fi
1258
1259	openssl rsautl -pubin -inkey pub.ssl -verify		\
1260	    < latest.ssl > tag.new 2>${QUIETREDIR} || true
1261	rm latest.ssl
1262
1263	if ! [ `wc -l < tag.new` = 1 ] ||
1264	    ! grep -qE	\
1265    "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
1266		tag.new; then
1267		echo "invalid signature."
1268		return 1
1269	fi
1270
1271	echo "done."
1272
1273	RELPATCHNUM=`cut -f 4 -d '|' < tag.new`
1274	TINDEXHASH=`cut -f 5 -d '|' < tag.new`
1275	EOLTIME=`cut -f 6 -d '|' < tag.new`
1276}
1277
1278# Sanity-check the patch number in a tag, to make sure that we're not
1279# going to "update" backwards and to prevent replay attacks.
1280fetch_tagsanity () {
1281	# Check that we're not going to move from -pX to -pY with Y < X.
1282	RELPX=`uname -r | sed -E 's,.*-,,'`
1283	if echo ${RELPX} | grep -qE '^p[0-9]+$'; then
1284		RELPX=`echo ${RELPX} | cut -c 2-`
1285	else
1286		RELPX=0
1287	fi
1288	if [ "${RELPATCHNUM}" -lt "${RELPX}" ]; then
1289		echo
1290		echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
1291		echo " appear older than what"
1292		echo "we are currently running (`uname -r`)!"
1293		echo "Cowardly refusing to proceed any further."
1294		return 1
1295	fi
1296
1297	# If "tag" exists and corresponds to ${RELNUM}, make sure that
1298	# it contains a patch number <= RELPATCHNUM, in order to protect
1299	# against rollback (replay) attacks.
1300	if [ -f tag ] &&
1301	    grep -qE	\
1302    "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
1303		tag; then
1304		LASTRELPATCHNUM=`cut -f 4 -d '|' < tag`
1305
1306		if [ "${RELPATCHNUM}" -lt "${LASTRELPATCHNUM}" ]; then
1307			echo
1308			echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
1309			echo " are older than the"
1310			echo -n "most recently seen updates"
1311			echo " (${RELNUM}-p${LASTRELPATCHNUM})."
1312			echo "Cowardly refusing to proceed any further."
1313			return 1
1314		fi
1315	fi
1316}
1317
1318# Fetch metadata index file
1319fetch_metadata_index () {
1320	echo ${NDEBUG} "Fetching metadata index... "
1321	rm -f ${TINDEXHASH}
1322	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/t/${TINDEXHASH}
1323	    2>${QUIETREDIR}
1324	if ! [ -f ${TINDEXHASH} ]; then
1325		echo "failed."
1326		return 1
1327	fi
1328	if [ `${SHA256} -q ${TINDEXHASH}` != ${TINDEXHASH} ]; then
1329		echo "update metadata index corrupt."
1330		return 1
1331	fi
1332	echo "done."
1333}
1334
1335# Print an error message about signed metadata being bogus.
1336fetch_metadata_bogus () {
1337	echo
1338	echo "The update metadata$1 is correctly signed, but"
1339	echo "failed an integrity check."
1340	echo "Cowardly refusing to proceed any further."
1341	return 1
1342}
1343
1344# Construct tINDEX.new by merging the lines named in $1 from ${TINDEXHASH}
1345# with the lines not named in $@ from tINDEX.present (if that file exists).
1346fetch_metadata_index_merge () {
1347	for METAFILE in $@; do
1348		if [ `grep -E "^${METAFILE}\|" ${TINDEXHASH} | wc -l`	\
1349		    -ne 1 ]; then
1350			fetch_metadata_bogus " index"
1351			return 1
1352		fi
1353
1354		grep -E "${METAFILE}\|" ${TINDEXHASH}
1355	done |
1356	    sort > tINDEX.wanted
1357
1358	if [ -f tINDEX.present ]; then
1359		join -t '|' -v 2 tINDEX.wanted tINDEX.present |
1360		    sort -m - tINDEX.wanted > tINDEX.new
1361		rm tINDEX.wanted
1362	else
1363		mv tINDEX.wanted tINDEX.new
1364	fi
1365}
1366
1367# Sanity check all the lines of tINDEX.new.  Even if more metadata lines
1368# are added by future versions of the server, this won't cause problems,
1369# since the only lines which appear in tINDEX.new are the ones which we
1370# specifically grepped out of ${TINDEXHASH}.
1371fetch_metadata_index_sanity () {
1372	if grep -qvE '^[0-9A-Z.-]+\|[0-9a-f]{64}$' tINDEX.new; then
1373		fetch_metadata_bogus " index"
1374		return 1
1375	fi
1376}
1377
1378# Sanity check the metadata file $1.
1379fetch_metadata_sanity () {
1380	# Some aliases to save space later: ${P} is a character which can
1381	# appear in a path; ${M} is the four numeric metadata fields; and
1382	# ${H} is a sha256 hash.
1383	P="[-+./:=,%@_[~[:alnum:]]"
1384	M="[0-9]+\|[0-9]+\|[0-9]+\|[0-9]+"
1385	H="[0-9a-f]{64}"
1386
1387	# Check that the first four fields make sense.
1388	if gunzip -c < files/$1.gz |
1389	    grep -qvE "^[a-z]+\|[0-9a-z-]+\|${P}+\|[fdL-]\|"; then
1390		fetch_metadata_bogus ""
1391		return 1
1392	fi
1393
1394	# Remove the first three fields.
1395	gunzip -c < files/$1.gz |
1396	    cut -f 4- -d '|' > sanitycheck.tmp
1397
1398	# Sanity check entries with type 'f'
1399	if grep -E '^f' sanitycheck.tmp |
1400	    grep -qvE "^f\|${M}\|${H}\|${P}*\$"; then
1401		fetch_metadata_bogus ""
1402		return 1
1403	fi
1404
1405	# Sanity check entries with type 'd'
1406	if grep -E '^d' sanitycheck.tmp |
1407	    grep -qvE "^d\|${M}\|\|\$"; then
1408		fetch_metadata_bogus ""
1409		return 1
1410	fi
1411
1412	# Sanity check entries with type 'L'
1413	if grep -E '^L' sanitycheck.tmp |
1414	    grep -qvE "^L\|${M}\|${P}*\|\$"; then
1415		fetch_metadata_bogus ""
1416		return 1
1417	fi
1418
1419	# Sanity check entries with type '-'
1420	if grep -E '^-' sanitycheck.tmp |
1421	    grep -qvE "^-\|\|\|\|\|\|"; then
1422		fetch_metadata_bogus ""
1423		return 1
1424	fi
1425
1426	# Clean up
1427	rm sanitycheck.tmp
1428}
1429
1430# Fetch the metadata index and metadata files listed in $@,
1431# taking advantage of metadata patches where possible.
1432fetch_metadata () {
1433	fetch_metadata_index || return 1
1434	fetch_metadata_index_merge $@ || return 1
1435	fetch_metadata_index_sanity || return 1
1436
1437	# Generate a list of wanted metadata patches
1438	join -t '|' -o 1.2,2.2 tINDEX.present tINDEX.new |
1439	    fetch_make_patchlist > patchlist
1440
1441	if [ -s patchlist ]; then
1442		# Attempt to fetch metadata patches
1443		echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
1444		echo ${NDEBUG} "metadata patches.${DDSTATS}"
1445		tr '|' '-' < patchlist |
1446		    lam -s "${FETCHDIR}/tp/" - -s ".gz" |
1447		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1448			2>${STATSREDIR} | fetch_progress
1449		echo "done."
1450
1451		# Attempt to apply metadata patches
1452		echo -n "Applying metadata patches... "
1453		tr '|' ' ' < patchlist |
1454		    while read X Y; do
1455			if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
1456			gunzip -c < ${X}-${Y}.gz > diff
1457			gunzip -c < files/${X}.gz > diff-OLD
1458
1459			# Figure out which lines are being added and removed
1460			grep -E '^-' diff |
1461			    cut -c 2- |
1462			    while read PREFIX; do
1463				look "${PREFIX}" diff-OLD
1464			    done |
1465			    sort > diff-rm
1466			grep -E '^\+' diff |
1467			    cut -c 2- > diff-add
1468
1469			# Generate the new file
1470			comm -23 diff-OLD diff-rm |
1471			    sort - diff-add > diff-NEW
1472
1473			if [ `${SHA256} -q diff-NEW` = ${Y} ]; then
1474				mv diff-NEW files/${Y}
1475				gzip -n files/${Y}
1476			else
1477				mv diff-NEW ${Y}.bad
1478			fi
1479			rm -f ${X}-${Y}.gz diff
1480			rm -f diff-OLD diff-NEW diff-add diff-rm
1481		done 2>${QUIETREDIR}
1482		echo "done."
1483	fi
1484
1485	# Update metadata without patches
1486	cut -f 2 -d '|' < tINDEX.new |
1487	    while read Y; do
1488		if [ ! -f "files/${Y}.gz" ]; then
1489			echo ${Y};
1490		fi
1491	    done |
1492	    sort -u > filelist
1493
1494	if [ -s filelist ]; then
1495		echo -n "Fetching `wc -l < filelist | tr -d ' '` "
1496		echo ${NDEBUG} "metadata files... "
1497		lam -s "${FETCHDIR}/m/" - -s ".gz" < filelist |
1498		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1499		    2>${QUIETREDIR}
1500
1501		while read Y; do
1502			if ! [ -f ${Y}.gz ]; then
1503				echo "failed."
1504				return 1
1505			fi
1506			if [ `gunzip -c < ${Y}.gz |
1507			    ${SHA256} -q` = ${Y} ]; then
1508				mv ${Y}.gz files/${Y}.gz
1509			else
1510				echo "metadata is corrupt."
1511				return 1
1512			fi
1513		done < filelist
1514		echo "done."
1515	fi
1516
1517# Sanity-check the metadata files.
1518	cut -f 2 -d '|' tINDEX.new > filelist
1519	while read X; do
1520		fetch_metadata_sanity ${X} || return 1
1521	done < filelist
1522
1523# Remove files which are no longer needed
1524	cut -f 2 -d '|' tINDEX.present |
1525	    sort > oldfiles
1526	cut -f 2 -d '|' tINDEX.new |
1527	    sort |
1528	    comm -13 - oldfiles |
1529	    lam -s "files/" - -s ".gz" |
1530	    xargs rm -f
1531	rm patchlist filelist oldfiles
1532	rm ${TINDEXHASH}
1533
1534# We're done!
1535	mv tINDEX.new tINDEX.present
1536	mv tag.new tag
1537
1538	return 0
1539}
1540
1541# Extract a subset of a downloaded metadata file containing only the parts
1542# which are listed in COMPONENTS.
1543fetch_filter_metadata_components () {
1544	METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
1545	gunzip -c < files/${METAHASH}.gz > $1.all
1546
1547	# Fish out the lines belonging to components we care about.
1548	for C in ${COMPONENTS}; do
1549		look "`echo ${C} | tr '/' '|'`|" $1.all
1550	done > $1
1551
1552	# Remove temporary file.
1553	rm $1.all
1554}
1555
1556# Generate a filtered version of the metadata file $1 from the downloaded
1557# file, by fishing out the lines corresponding to components we're trying
1558# to keep updated, and then removing lines corresponding to paths we want
1559# to ignore.
1560fetch_filter_metadata () {
1561	# Fish out the lines belonging to components we care about.
1562	fetch_filter_metadata_components $1
1563
1564	# Canonicalize directory names by removing any trailing / in
1565	# order to avoid listing directories multiple times if they
1566	# belong to multiple components.  Turning "/" into "" doesn't
1567	# matter, since we add a leading "/" when we use paths later.
1568	cut -f 3- -d '|' $1 |
1569	    sed -e 's,/|d|,|d|,' |
1570	    sed -e 's,/|-|,|-|,' |
1571	    sort -u > $1.tmp
1572
1573	# Figure out which lines to ignore and remove them.
1574	for X in ${IGNOREPATHS}; do
1575		grep -E "^${X}" $1.tmp
1576	done |
1577	    sort -u |
1578	    comm -13 - $1.tmp > $1
1579
1580	# Remove temporary files.
1581	rm $1.tmp
1582}
1583
1584# Filter the metadata file $1 by adding lines with "/boot/$2"
1585# replaced by ${KERNELDIR} (which is `sysctl -n kern.bootfile` minus the
1586# trailing "/kernel"); and if "/boot/$2" does not exist, remove
1587# the original lines which start with that.
1588# Put another way: Deal with the fact that the FOO kernel is sometimes
1589# installed in /boot/FOO/ and is sometimes installed elsewhere.
1590fetch_filter_kernel_names () {
1591	grep ^/boot/$2 $1 |
1592	    sed -e "s,/boot/$2,${KERNELDIR},g" |
1593	    sort - $1 > $1.tmp
1594	mv $1.tmp $1
1595
1596	if ! [ -d /boot/$2 ]; then
1597		grep -v ^/boot/$2 $1 > $1.tmp
1598		mv $1.tmp $1
1599	fi
1600}
1601
1602# For all paths appearing in $1 or $3, inspect the system
1603# and generate $2 describing what is currently installed.
1604fetch_inspect_system () {
1605	# No errors yet...
1606	rm -f .err
1607
1608	# Tell the user why his disk is suddenly making lots of noise
1609	echo -n "Inspecting system... "
1610
1611	# Generate list of files to inspect
1612	cat $1 $3 |
1613	    cut -f 1 -d '|' |
1614	    sort -u > filelist
1615
1616	# Examine each file and output lines of the form
1617	# /path/to/file|type|device-inum|user|group|perm|flags|value
1618	# sorted by device and inode number.
1619	while read F; do
1620		# If the symlink/file/directory does not exist, record this.
1621		if ! [ -e ${BASEDIR}/${F} ]; then
1622			echo "${F}|-||||||"
1623			continue
1624		fi
1625		if ! [ -r ${BASEDIR}/${F} ]; then
1626			echo "Cannot read file: ${BASEDIR}/${F}"	\
1627			    >/dev/stderr
1628			touch .err
1629			return 1
1630		fi
1631
1632		# Otherwise, output an index line.
1633		if [ -L ${BASEDIR}/${F} ]; then
1634			echo -n "${F}|L|"
1635			stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1636			readlink ${BASEDIR}/${F};
1637		elif [ -f ${BASEDIR}/${F} ]; then
1638			echo -n "${F}|f|"
1639			stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1640			sha256 -q ${BASEDIR}/${F};
1641		elif [ -d ${BASEDIR}/${F} ]; then
1642			echo -n "${F}|d|"
1643			stat -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1644		else
1645			echo "Unknown file type: ${BASEDIR}/${F}"	\
1646			    >/dev/stderr
1647			touch .err
1648			return 1
1649		fi
1650	done < filelist |
1651	    sort -k 3,3 -t '|' > $2.tmp
1652	rm filelist
1653
1654	# Check if an error occurred during system inspection
1655	if [ -f .err ]; then
1656		return 1
1657	fi
1658
1659	# Convert to the form
1660	# /path/to/file|type|user|group|perm|flags|value|hlink
1661	# by resolving identical device and inode numbers into hard links.
1662	cut -f 1,3 -d '|' $2.tmp |
1663	    sort -k 1,1 -t '|' |
1664	    sort -s -u -k 2,2 -t '|' |
1665	    join -1 2 -2 3 -t '|' - $2.tmp |
1666	    awk -F \| -v OFS=\|		\
1667		'{
1668		    if (($2 == $3) || ($4 == "-"))
1669			print $3,$4,$5,$6,$7,$8,$9,""
1670		    else
1671			print $3,$4,$5,$6,$7,$8,$9,$2
1672		}' |
1673	    sort > $2
1674	rm $2.tmp
1675
1676	# We're finished looking around
1677	echo "done."
1678}
1679
1680# For any paths matching ${MERGECHANGES}, compare $2 against $1 and $3 and
1681# find any files with values unique to $2; generate $4 containing these paths
1682# and their corresponding hashes from $1.
1683fetch_filter_mergechanges () {
1684	# Pull out the paths and hashes of the files matching ${MERGECHANGES}.
1685	for F in $1 $2 $3; do
1686		for X in ${MERGECHANGES}; do
1687			grep -E "^${X}" ${F}
1688		done |
1689		    cut -f 1,2,7 -d '|' |
1690		    sort > ${F}-values
1691	done
1692
1693	# Any line in $2-values which doesn't appear in $1-values or $3-values
1694	# and is a file means that we should list the path in $3.
1695	sort $1-values $3-values |
1696	    comm -13 - $2-values |
1697	    fgrep '|f|' |
1698	    cut -f 1 -d '|' > $2-paths
1699
1700	# For each path, pull out one (and only one!) entry from $1-values.
1701	# Note that we cannot distinguish which "old" version the user made
1702	# changes to; but hopefully any changes which occur due to security
1703	# updates will exist in both the "new" version and the version which
1704	# the user has installed, so the merging will still work.
1705	while read X; do
1706		look "${X}|" $1-values |
1707		    head -1
1708	done < $2-paths > $4
1709
1710	# Clean up
1711	rm $1-values $2-values $3-values $2-paths
1712}
1713
1714# For any paths matching ${UPDATEIFUNMODIFIED}, remove lines from $[123]
1715# which correspond to lines in $2 with hashes not matching $1 or $3, unless
1716# the paths are listed in $4.  For entries in $2 marked "not present"
1717# (aka. type -), remove lines from $[123] unless there is a corresponding
1718# entry in $1.
1719fetch_filter_unmodified_notpresent () {
1720	# Figure out which lines of $1 and $3 correspond to bits which
1721	# should only be updated if they haven't changed, and fish out
1722	# the (path, type, value) tuples.
1723	# NOTE: We don't consider a file to be "modified" if it matches
1724	# the hash from $3.
1725	for X in ${UPDATEIFUNMODIFIED}; do
1726		grep -E "^${X}" $1
1727		grep -E "^${X}" $3
1728	done |
1729	    cut -f 1,2,7 -d '|' |
1730	    sort > $1-values
1731
1732	# Do the same for $2.
1733	for X in ${UPDATEIFUNMODIFIED}; do
1734		grep -E "^${X}" $2
1735	done |
1736	    cut -f 1,2,7 -d '|' |
1737	    sort > $2-values
1738
1739	# Any entry in $2-values which is not in $1-values corresponds to
1740	# a path which we need to remove from $1, $2, and $3, unless it
1741	# that path appears in $4.
1742	comm -13 $1-values $2-values |
1743	    sort -t '|' -k 1,1 > mlines.tmp
1744	cut -f 1 -d '|' $4 |
1745	    sort |
1746	    join -v 2 -t '|' - mlines.tmp |
1747	    sort > mlines
1748	rm $1-values $2-values mlines.tmp
1749
1750	# Any lines in $2 which are not in $1 AND are "not present" lines
1751	# also belong in mlines.
1752	comm -13 $1 $2 |
1753	    cut -f 1,2,7 -d '|' |
1754	    fgrep '|-|' >> mlines
1755
1756	# Remove lines from $1, $2, and $3
1757	for X in $1 $2 $3; do
1758		sort -t '|' -k 1,1 ${X} > ${X}.tmp
1759		cut -f 1 -d '|' < mlines |
1760		    sort |
1761		    join -v 2 -t '|' - ${X}.tmp |
1762		    sort > ${X}
1763		rm ${X}.tmp
1764	done
1765
1766	# Store a list of the modified files, for future reference
1767	fgrep -v '|-|' mlines |
1768	    cut -f 1 -d '|' > modifiedfiles
1769	rm mlines
1770}
1771
1772# For each entry in $1 of type -, remove any corresponding
1773# entry from $2 if ${ALLOWADD} != "yes".  Remove all entries
1774# of type - from $1.
1775fetch_filter_allowadd () {
1776	cut -f 1,2 -d '|' < $1 |
1777	    fgrep '|-' |
1778	    cut -f 1 -d '|' > filesnotpresent
1779
1780	if [ ${ALLOWADD} != "yes" ]; then
1781		sort < $2 |
1782		    join -v 1 -t '|' - filesnotpresent |
1783		    sort > $2.tmp
1784		mv $2.tmp $2
1785	fi
1786
1787	sort < $1 |
1788	    join -v 1 -t '|' - filesnotpresent |
1789	    sort > $1.tmp
1790	mv $1.tmp $1
1791	rm filesnotpresent
1792}
1793
1794# If ${ALLOWDELETE} != "yes", then remove any entries from $1
1795# which don't correspond to entries in $2.
1796fetch_filter_allowdelete () {
1797	# Produce a lists ${PATH}|${TYPE}
1798	for X in $1 $2; do
1799		cut -f 1-2 -d '|' < ${X} |
1800		    sort -u > ${X}.nodes
1801	done
1802
1803	# Figure out which lines need to be removed from $1.
1804	if [ ${ALLOWDELETE} != "yes" ]; then
1805		comm -23 $1.nodes $2.nodes > $1.badnodes
1806	else
1807		: > $1.badnodes
1808	fi
1809
1810	# Remove the relevant lines from $1
1811	while read X; do
1812		look "${X}|" $1
1813	done < $1.badnodes |
1814	    comm -13 - $1 > $1.tmp
1815	mv $1.tmp $1
1816
1817	rm $1.badnodes $1.nodes $2.nodes
1818}
1819
1820# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in $2
1821# with metadata not matching any entry in $1, replace the corresponding
1822# line of $3 with one having the same metadata as the entry in $2.
1823fetch_filter_modified_metadata () {
1824	# Fish out the metadata from $1 and $2
1825	for X in $1 $2; do
1826		cut -f 1-6 -d '|' < ${X} > ${X}.metadata
1827	done
1828
1829	# Find the metadata we need to keep
1830	if [ ${KEEPMODIFIEDMETADATA} = "yes" ]; then
1831		comm -13 $1.metadata $2.metadata > keepmeta
1832	else
1833		: > keepmeta
1834	fi
1835
1836	# Extract the lines which we need to remove from $3, and
1837	# construct the lines which we need to add to $3.
1838	: > $3.remove
1839	: > $3.add
1840	while read LINE; do
1841		NODE=`echo "${LINE}" | cut -f 1-2 -d '|'`
1842		look "${NODE}|" $3 >> $3.remove
1843		look "${NODE}|" $3 |
1844		    cut -f 7- -d '|' |
1845		    lam -s "${LINE}|" - >> $3.add
1846	done < keepmeta
1847
1848	# Remove the specified lines and add the new lines.
1849	sort $3.remove |
1850	    comm -13 - $3 |
1851	    sort -u - $3.add > $3.tmp
1852	mv $3.tmp $3
1853
1854	rm keepmeta $1.metadata $2.metadata $3.add $3.remove
1855}
1856
1857# Remove lines from $1 and $2 which are identical;
1858# no need to update a file if it isn't changing.
1859fetch_filter_uptodate () {
1860	comm -23 $1 $2 > $1.tmp
1861	comm -13 $1 $2 > $2.tmp
1862
1863	mv $1.tmp $1
1864	mv $2.tmp $2
1865}
1866
1867# Fetch any "clean" old versions of files we need for merging changes.
1868fetch_files_premerge () {
1869	# We only need to do anything if $1 is non-empty.
1870	if [ -s $1 ]; then
1871		# Tell the user what we're doing
1872		echo -n "Fetching files from ${OLDRELNUM} for merging... "
1873
1874		# List of files wanted
1875		fgrep '|f|' < $1 |
1876		    cut -f 3 -d '|' |
1877		    sort -u > files.wanted
1878
1879		# Only fetch the files we don't already have
1880		while read Y; do
1881			if [ ! -f "files/${Y}.gz" ]; then
1882				echo ${Y};
1883			fi
1884		done < files.wanted > filelist
1885
1886		# Actually fetch them
1887		lam -s "${OLDFETCHDIR}/f/" - -s ".gz" < filelist |
1888		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1889		    2>${QUIETREDIR}
1890
1891		# Make sure we got them all, and move them into /files/
1892		while read Y; do
1893			if ! [ -f ${Y}.gz ]; then
1894				echo "failed."
1895				return 1
1896			fi
1897			if [ `gunzip -c < ${Y}.gz |
1898			    ${SHA256} -q` = ${Y} ]; then
1899				mv ${Y}.gz files/${Y}.gz
1900			else
1901				echo "${Y} has incorrect hash."
1902				return 1
1903			fi
1904		done < filelist
1905		echo "done."
1906
1907		# Clean up
1908		rm filelist files.wanted
1909	fi
1910}
1911
1912# Prepare to fetch files: Generate a list of the files we need,
1913# copy the unmodified files we have into /files/, and generate
1914# a list of patches to download.
1915fetch_files_prepare () {
1916	# Tell the user why his disk is suddenly making lots of noise
1917	echo -n "Preparing to download files... "
1918
1919	# Reduce indices to ${PATH}|${HASH} pairs
1920	for X in $1 $2 $3; do
1921		cut -f 1,2,7 -d '|' < ${X} |
1922		    fgrep '|f|' |
1923		    cut -f 1,3 -d '|' |
1924		    sort > ${X}.hashes
1925	done
1926
1927	# List of files wanted
1928	cut -f 2 -d '|' < $3.hashes |
1929	    sort -u |
1930	    while read HASH; do
1931		if ! [ -f files/${HASH}.gz ]; then
1932			echo ${HASH}
1933		fi
1934	done > files.wanted
1935
1936	# Generate a list of unmodified files
1937	comm -12 $1.hashes $2.hashes |
1938	    sort -k 1,1 -t '|' > unmodified.files
1939
1940	# Copy all files into /files/.  We only need the unmodified files
1941	# for use in patching; but we'll want all of them if the user asks
1942	# to rollback the updates later.
1943	while read LINE; do
1944		F=`echo "${LINE}" | cut -f 1 -d '|'`
1945		HASH=`echo "${LINE}" | cut -f 2 -d '|'`
1946
1947		# Skip files we already have.
1948		if [ -f files/${HASH}.gz ]; then
1949			continue
1950		fi
1951
1952		# Make sure the file hasn't changed.
1953		cp "${BASEDIR}/${F}" tmpfile
1954		if [ `sha256 -q tmpfile` != ${HASH} ]; then
1955			echo
1956			echo "File changed while FreeBSD Update running: ${F}"
1957			return 1
1958		fi
1959
1960		# Place the file into storage.
1961		gzip -c < tmpfile > files/${HASH}.gz
1962		rm tmpfile
1963	done < $2.hashes
1964
1965	# Produce a list of patches to download
1966	sort -k 1,1 -t '|' $3.hashes |
1967	    join -t '|' -o 2.2,1.2 - unmodified.files |
1968	    fetch_make_patchlist > patchlist
1969
1970	# Garbage collect
1971	rm unmodified.files $1.hashes $2.hashes $3.hashes
1972
1973	# We don't need the list of possible old files any more.
1974	rm $1
1975
1976	# We're finished making noise
1977	echo "done."
1978}
1979
1980# Fetch files.
1981fetch_files () {
1982	# Attempt to fetch patches
1983	if [ -s patchlist ]; then
1984		echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
1985		echo ${NDEBUG} "patches.${DDSTATS}"
1986		tr '|' '-' < patchlist |
1987		    lam -s "${PATCHDIR}/" - |
1988		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1989			2>${STATSREDIR} | fetch_progress
1990		echo "done."
1991
1992		# Attempt to apply patches
1993		echo -n "Applying patches... "
1994		tr '|' ' ' < patchlist |
1995		    while read X Y; do
1996			if [ ! -f "${X}-${Y}" ]; then continue; fi
1997			gunzip -c < files/${X}.gz > OLD
1998
1999			bspatch OLD NEW ${X}-${Y}
2000
2001			if [ `${SHA256} -q NEW` = ${Y} ]; then
2002				mv NEW files/${Y}
2003				gzip -n files/${Y}
2004			fi
2005			rm -f diff OLD NEW ${X}-${Y}
2006		done 2>${QUIETREDIR}
2007		echo "done."
2008	fi
2009
2010	# Download files which couldn't be generate via patching
2011	while read Y; do
2012		if [ ! -f "files/${Y}.gz" ]; then
2013			echo ${Y};
2014		fi
2015	done < files.wanted > filelist
2016
2017	if [ -s filelist ]; then
2018		echo -n "Fetching `wc -l < filelist | tr -d ' '` "
2019		echo ${NDEBUG} "files... "
2020		lam -s "${FETCHDIR}/f/" - -s ".gz" < filelist |
2021		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
2022			2>${STATSREDIR} | fetch_progress
2023
2024		while read Y; do
2025			if ! [ -f ${Y}.gz ]; then
2026				echo "failed."
2027				return 1
2028			fi
2029			if [ `gunzip -c < ${Y}.gz |
2030			    ${SHA256} -q` = ${Y} ]; then
2031				mv ${Y}.gz files/${Y}.gz
2032			else
2033				echo "${Y} has incorrect hash."
2034				return 1
2035			fi
2036		done < filelist
2037		echo "done."
2038	fi
2039
2040	# Clean up
2041	rm files.wanted filelist patchlist
2042}
2043
2044# Create and populate install manifest directory; and report what updates
2045# are available.
2046fetch_create_manifest () {
2047	# If we have an existing install manifest, nuke it.
2048	if [ -L "${BDHASH}-install" ]; then
2049		rm -r ${BDHASH}-install/
2050		rm ${BDHASH}-install
2051	fi
2052
2053	# Report to the user if any updates were avoided due to local changes
2054	if [ -s modifiedfiles ]; then
2055		cat - modifiedfiles <<- EOF | ${PAGER}
2056			The following files are affected by updates. No changes have
2057			been downloaded, however, because the files have been modified
2058			locally:
2059		EOF
2060	fi
2061	rm modifiedfiles
2062
2063	# If no files will be updated, tell the user and exit
2064	if ! [ -s INDEX-PRESENT ] &&
2065	    ! [ -s INDEX-NEW ]; then
2066		rm INDEX-PRESENT INDEX-NEW
2067		echo
2068		echo -n "No updates needed to update system to "
2069		echo "${RELNUM}-p${RELPATCHNUM}."
2070		return
2071	fi
2072
2073	# Divide files into (a) removed files, (b) added files, and
2074	# (c) updated files.
2075	cut -f 1 -d '|' < INDEX-PRESENT |
2076	    sort > INDEX-PRESENT.flist
2077	cut -f 1 -d '|' < INDEX-NEW |
2078	    sort > INDEX-NEW.flist
2079	comm -23 INDEX-PRESENT.flist INDEX-NEW.flist > files.removed
2080	comm -13 INDEX-PRESENT.flist INDEX-NEW.flist > files.added
2081	comm -12 INDEX-PRESENT.flist INDEX-NEW.flist > files.updated
2082	rm INDEX-PRESENT.flist INDEX-NEW.flist
2083
2084	# Report removed files, if any
2085	if [ -s files.removed ]; then
2086		cat - files.removed <<- EOF | ${PAGER}
2087			The following files will be removed as part of updating to
2088			${RELNUM}-p${RELPATCHNUM}:
2089		EOF
2090	fi
2091	rm files.removed
2092
2093	# Report added files, if any
2094	if [ -s files.added ]; then
2095		cat - files.added <<- EOF | ${PAGER}
2096			The following files will be added as part of updating to
2097			${RELNUM}-p${RELPATCHNUM}:
2098		EOF
2099	fi
2100	rm files.added
2101
2102	# Report updated files, if any
2103	if [ -s files.updated ]; then
2104		cat - files.updated <<- EOF | ${PAGER}
2105			The following files will be updated as part of updating to
2106			${RELNUM}-p${RELPATCHNUM}:
2107		EOF
2108	fi
2109	rm files.updated
2110
2111	# Create a directory for the install manifest.
2112	MDIR=`mktemp -d install.XXXXXX` || return 1
2113
2114	# Populate it
2115	mv INDEX-PRESENT ${MDIR}/INDEX-OLD
2116	mv INDEX-NEW ${MDIR}/INDEX-NEW
2117
2118	# Link it into place
2119	ln -s ${MDIR} ${BDHASH}-install
2120}
2121
2122# Warn about any upcoming EoL
2123fetch_warn_eol () {
2124	# What's the current time?
2125	NOWTIME=`date "+%s"`
2126
2127	# When did we last warn about the EoL date?
2128	if [ -f lasteolwarn ]; then
2129		LASTWARN=`cat lasteolwarn`
2130	else
2131		LASTWARN=`expr ${NOWTIME} - 63072000`
2132	fi
2133
2134	# If the EoL time is past, warn.
2135	if [ ${EOLTIME} -lt ${NOWTIME} ]; then
2136		echo
2137		cat <<-EOF
2138		WARNING: `uname -sr` HAS PASSED ITS END-OF-LIFE DATE.
2139		Any security issues discovered after `date -r ${EOLTIME}`
2140		will not have been corrected.
2141		EOF
2142		return 1
2143	fi
2144
2145	# Figure out how long it has been since we last warned about the
2146	# upcoming EoL, and how much longer we have left.
2147	SINCEWARN=`expr ${NOWTIME} - ${LASTWARN}`
2148	TIMELEFT=`expr ${EOLTIME} - ${NOWTIME}`
2149
2150	# Don't warn if the EoL is more than 3 months away
2151	if [ ${TIMELEFT} -gt 7884000 ]; then
2152		return 0
2153	fi
2154
2155	# Don't warn if the time remaining is more than 3 times the time
2156	# since the last warning.
2157	if [ ${TIMELEFT} -gt `expr ${SINCEWARN} \* 3` ]; then
2158		return 0
2159	fi
2160
2161	# Figure out what time units to use.
2162	if [ ${TIMELEFT} -lt 604800 ]; then
2163		UNIT="day"
2164		SIZE=86400
2165	elif [ ${TIMELEFT} -lt 2678400 ]; then
2166		UNIT="week"
2167		SIZE=604800
2168	else
2169		UNIT="month"
2170		SIZE=2678400
2171	fi
2172
2173	# Compute the right number of units
2174	NUM=`expr ${TIMELEFT} / ${SIZE}`
2175	if [ ${NUM} != 1 ]; then
2176		UNIT="${UNIT}s"
2177	fi
2178
2179	# Print the warning
2180	echo
2181	cat <<-EOF
2182		WARNING: `uname -sr` is approaching its End-of-Life date.
2183		It is strongly recommended that you upgrade to a newer
2184		release within the next ${NUM} ${UNIT}.
2185	EOF
2186
2187	# Update the stored time of last warning
2188	echo ${NOWTIME} > lasteolwarn
2189}
2190
2191# Do the actual work involved in "fetch" / "cron".
2192fetch_run () {
2193	workdir_init || return 1
2194
2195	# Prepare the mirror list.
2196	fetch_pick_server_init && fetch_pick_server
2197
2198	# Try to fetch the public key until we run out of servers.
2199	while ! fetch_key; do
2200		fetch_pick_server || return 1
2201	done
2202
2203	# Try to fetch the metadata index signature ("tag") until we run
2204	# out of available servers; and sanity check the downloaded tag.
2205	while ! fetch_tag; do
2206		fetch_pick_server || return 1
2207	done
2208	fetch_tagsanity || return 1
2209
2210	# Fetch the latest INDEX-NEW and INDEX-OLD files.
2211	fetch_metadata INDEX-NEW INDEX-OLD || return 1
2212
2213	# Generate filtered INDEX-NEW and INDEX-OLD files containing only
2214	# the lines which (a) belong to components we care about, and (b)
2215	# don't correspond to paths we're explicitly ignoring.
2216	fetch_filter_metadata INDEX-NEW || return 1
2217	fetch_filter_metadata INDEX-OLD || return 1
2218
2219	# Translate /boot/${KERNCONF} into ${KERNELDIR}
2220	fetch_filter_kernel_names INDEX-NEW ${KERNCONF}
2221	fetch_filter_kernel_names INDEX-OLD ${KERNCONF}
2222
2223	# For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
2224	# system and generate an INDEX-PRESENT file.
2225	fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2226
2227	# Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
2228	# correspond to lines in INDEX-PRESENT with hashes not appearing
2229	# in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
2230	# INDEX-PRESENT has type - and there isn't a corresponding entry in
2231	# INDEX-OLD with type -.
2232	fetch_filter_unmodified_notpresent	\
2233	    INDEX-OLD INDEX-PRESENT INDEX-NEW /dev/null
2234
2235	# For each entry in INDEX-PRESENT of type -, remove any corresponding
2236	# entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
2237	# of type - from INDEX-PRESENT.
2238	fetch_filter_allowadd INDEX-PRESENT INDEX-NEW
2239
2240	# If ${ALLOWDELETE} != "yes", then remove any entries from
2241	# INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
2242	fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW
2243
2244	# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
2245	# INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
2246	# replace the corresponding line of INDEX-NEW with one having the
2247	# same metadata as the entry in INDEX-PRESENT.
2248	fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW
2249
2250	# Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
2251	# no need to update a file if it isn't changing.
2252	fetch_filter_uptodate INDEX-PRESENT INDEX-NEW
2253
2254	# Prepare to fetch files: Generate a list of the files we need,
2255	# copy the unmodified files we have into /files/, and generate
2256	# a list of patches to download.
2257	fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2258
2259	# Fetch files.
2260	fetch_files || return 1
2261
2262	# Create and populate install manifest directory; and report what
2263	# updates are available.
2264	fetch_create_manifest || return 1
2265
2266	# Warn about any upcoming EoL
2267	fetch_warn_eol || return 1
2268}
2269
2270# If StrictComponents is not "yes", generate a new components list
2271# with only the components which appear to be installed.
2272upgrade_guess_components () {
2273	if [ "${STRICTCOMPONENTS}" = "no" ]; then
2274		# Generate filtered INDEX-ALL with only the components listed
2275		# in COMPONENTS.
2276		fetch_filter_metadata_components $1 || return 1
2277
2278		# Tell the user why his disk is suddenly making lots of noise
2279		echo -n "Inspecting system... "
2280
2281		# Look at the files on disk, and assume that a component is
2282		# supposed to be present if it is more than half-present.
2283		cut -f 1-3 -d '|' < INDEX-ALL |
2284		    tr '|' ' ' |
2285		    while read C S F; do
2286			if [ -e ${BASEDIR}/${F} ]; then
2287				echo "+ ${C}|${S}"
2288			fi
2289			echo "= ${C}|${S}"
2290		    done |
2291		    sort |
2292		    uniq -c |
2293		    sed -E 's,^ +,,' > compfreq
2294		grep ' = ' compfreq |
2295		    cut -f 1,3 -d ' ' |
2296		    sort -k 2,2 -t ' ' > compfreq.total
2297		grep ' + ' compfreq |
2298		    cut -f 1,3 -d ' ' |
2299		    sort -k 2,2 -t ' ' > compfreq.present
2300		join -t ' ' -1 2 -2 2 compfreq.present compfreq.total |
2301		    while read S P T; do
2302			if [ ${T} -ne 0 -a ${P} -gt `expr ${T} / 2` ]; then
2303				echo ${S}
2304			fi
2305		    done > comp.present
2306		cut -f 2 -d ' ' < compfreq.total > comp.total
2307		rm INDEX-ALL compfreq compfreq.total compfreq.present
2308
2309		# We're done making noise.
2310		echo "done."
2311
2312		# Sometimes the kernel isn't installed where INDEX-ALL
2313		# thinks that it should be: In particular, it is often in
2314		# /boot/kernel instead of /boot/GENERIC or /boot/SMP.  To
2315		# deal with this, if "kernel|X" is listed in comp.total
2316		# (i.e., is a component which would be upgraded if it is
2317		# found to be present) we will add it to comp.present.
2318		# If "kernel|<anything>" is in comp.total but "kernel|X" is
2319		# not, we print a warning -- the user is running a kernel
2320		# which isn't part of the release.
2321		KCOMP=`echo ${KERNCONF} | tr 'A-Z' 'a-z'`
2322		grep -E "^kernel\|${KCOMP}\$" comp.total >> comp.present
2323
2324		if grep -qE "^kernel\|" comp.total &&
2325		    ! grep -qE "^kernel\|${KCOMP}\$" comp.total; then
2326			cat <<-EOF
2327
2328WARNING: This system is running a "${KCOMP}" kernel, which is not a
2329kernel configuration distributed as part of FreeBSD ${RELNUM}.
2330This kernel will not be updated: you MUST update the kernel manually
2331before running "$0 install".
2332			EOF
2333		fi
2334
2335		# Re-sort the list of installed components and generate
2336		# the list of non-installed components.
2337		sort -u < comp.present > comp.present.tmp
2338		mv comp.present.tmp comp.present
2339		comm -13 comp.present comp.total > comp.absent
2340
2341		# Ask the user to confirm that what we have is correct.  To
2342		# reduce user confusion, translate "X|Y" back to "X/Y" (as
2343		# subcomponents must be listed in the configuration file).
2344		echo
2345		echo -n "The following components of FreeBSD "
2346		echo "seem to be installed:"
2347		tr '|' '/' < comp.present |
2348		    fmt -72
2349		echo
2350		echo -n "The following components of FreeBSD "
2351		echo "do not seem to be installed:"
2352		tr '|' '/' < comp.absent |
2353		    fmt -72
2354		echo
2355		continuep || return 1
2356		echo
2357
2358		# Suck the generated list of components into ${COMPONENTS}.
2359		# Note that comp.present.tmp is used due to issues with
2360		# pipelines and setting variables.
2361		COMPONENTS=""
2362		tr '|' '/' < comp.present > comp.present.tmp
2363		while read C; do
2364			COMPONENTS="${COMPONENTS} ${C}"
2365		done < comp.present.tmp
2366
2367		# Delete temporary files
2368		rm comp.present comp.present.tmp comp.absent comp.total
2369	fi
2370}
2371
2372# If StrictComponents is not "yes", COMPONENTS contains an entry
2373# corresponding to the currently running kernel, and said kernel
2374# does not exist in the new release, add "kernel/generic" to the
2375# list of components.
2376upgrade_guess_new_kernel () {
2377	if [ "${STRICTCOMPONENTS}" = "no" ]; then
2378		# Grab the unfiltered metadata file.
2379		METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
2380		gunzip -c < files/${METAHASH}.gz > $1.all
2381
2382		# If "kernel/${KCOMP}" is in ${COMPONENTS} and that component
2383		# isn't in $1.all, we need to add kernel/generic.
2384		for C in ${COMPONENTS}; do
2385			if [ ${C} = "kernel/${KCOMP}" ] &&
2386			    ! grep -qE "^kernel\|${KCOMP}\|" $1.all; then
2387				COMPONENTS="${COMPONENTS} kernel/generic"
2388				NKERNCONF="GENERIC"
2389				cat <<-EOF
2390
2391WARNING: This system is running a "${KCOMP}" kernel, which is not a
2392kernel configuration distributed as part of FreeBSD ${RELNUM}.
2393As part of upgrading to FreeBSD ${RELNUM}, this kernel will be
2394replaced with a "generic" kernel.
2395				EOF
2396				continuep || return 1
2397			fi
2398		done
2399
2400		# Don't need this any more...
2401		rm $1.all
2402	fi
2403}
2404
2405# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
2406# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
2407upgrade_oldall_to_oldnew () {
2408	# For each ${F}|... which appears in INDEX-ALL but does not appear
2409	# in INDEX-OLD, add ${F}|-|||||| to INDEX-OLD.
2410	cut -f 1 -d '|' < $1 |
2411	    sort -u > $1.paths
2412	cut -f 1 -d '|' < $2 |
2413	    sort -u |
2414	    comm -13 $1.paths - |
2415	    lam - -s "|-||||||" |
2416	    sort - $1 > $1.tmp
2417	mv $1.tmp $1
2418
2419	# Remove lines from INDEX-OLD which also appear in INDEX-ALL
2420	comm -23 $1 $2 > $1.tmp
2421	mv $1.tmp $1
2422
2423	# Remove lines from INDEX-ALL which have a file name not appearing
2424	# anywhere in INDEX-OLD (since these must be files which haven't
2425	# changed -- if they were new, there would be an entry of type "-").
2426	cut -f 1 -d '|' < $1 |
2427	    sort -u > $1.paths
2428	sort -k 1,1 -t '|' < $2 |
2429	    join -t '|' - $1.paths |
2430	    sort > $2.tmp
2431	rm $1.paths
2432	mv $2.tmp $2
2433
2434	# Rename INDEX-ALL to INDEX-NEW.
2435	mv $2 $3
2436}
2437
2438# Helper for upgrade_merge: Return zero true iff the two files differ only
2439# in the contents of their RCS tags.
2440samef () {
2441	X=`sed -E 's/\\$FreeBSD.*\\$/\$FreeBSD\$/' < $1 | ${SHA256}`
2442	Y=`sed -E 's/\\$FreeBSD.*\\$/\$FreeBSD\$/' < $2 | ${SHA256}`
2443
2444	if [ $X = $Y ]; then
2445		return 0;
2446	else
2447		return 1;
2448	fi
2449}
2450
2451# From the list of "old" files in $1, merge changes in $2 with those in $3,
2452# and update $3 to reflect the hashes of merged files.
2453upgrade_merge () {
2454	# We only need to do anything if $1 is non-empty.
2455	if [ -s $1 ]; then
2456		cut -f 1 -d '|' $1 |
2457		    sort > $1-paths
2458
2459		# Create staging area for merging files
2460		rm -rf merge/
2461		while read F; do
2462			D=`dirname ${F}`
2463			mkdir -p merge/old/${D}
2464			mkdir -p merge/${OLDRELNUM}/${D}
2465			mkdir -p merge/${RELNUM}/${D}
2466			mkdir -p merge/new/${D}
2467		done < $1-paths
2468
2469		# Copy in files
2470		while read F; do
2471			# Currently installed file
2472			V=`look "${F}|" $2 | cut -f 7 -d '|'`
2473			gunzip < files/${V}.gz > merge/old/${F}
2474
2475			# Old release
2476			if look "${F}|" $1 | fgrep -q "|f|"; then
2477				V=`look "${F}|" $1 | cut -f 3 -d '|'`
2478				gunzip < files/${V}.gz		\
2479				    > merge/${OLDRELNUM}/${F}
2480			fi
2481
2482			# New release
2483			if look "${F}|" $3 | cut -f 1,2,7 -d '|' |
2484			    fgrep -q "|f|"; then
2485				V=`look "${F}|" $3 | cut -f 7 -d '|'`
2486				gunzip < files/${V}.gz		\
2487				    > merge/${RELNUM}/${F}
2488			fi
2489		done < $1-paths
2490
2491		# Attempt to automatically merge changes
2492		echo -n "Attempting to automatically merge "
2493		echo -n "changes in files..."
2494		: > failed.merges
2495		while read F; do
2496			# If the file doesn't exist in the new release,
2497			# the result of "merging changes" is having the file
2498			# not exist.
2499			if ! [ -f merge/${RELNUM}/${F} ]; then
2500				continue
2501			fi
2502
2503			# If the file didn't exist in the old release, we're
2504			# going to throw away the existing file and hope that
2505			# the version from the new release is what we want.
2506			if ! [ -f merge/${OLDRELNUM}/${F} ]; then
2507				cp merge/${RELNUM}/${F} merge/new/${F}
2508				continue
2509			fi
2510
2511			# Some files need special treatment.
2512			case ${F} in
2513			/etc/spwd.db | /etc/pwd.db | /etc/login.conf.db)
2514				# Don't merge these -- we're rebuild them
2515				# after updates are installed.
2516				cp merge/old/${F} merge/new/${F}
2517				;;
2518			*)
2519				if ! diff3 -E -m -L "current version"	\
2520				    -L "${OLDRELNUM}" -L "${RELNUM}"	\
2521				    merge/old/${F}			\
2522				    merge/${OLDRELNUM}/${F}		\
2523				    merge/${RELNUM}/${F}		\
2524				    > merge/new/${F} 2>/dev/null; then
2525					echo ${F} >> failed.merges
2526				fi
2527				;;
2528			esac
2529		done < $1-paths
2530		echo " done."
2531
2532		# Ask the user to handle any files which didn't merge.
2533		while read F; do
2534			# If the installed file differs from the version in
2535			# the old release only due to RCS tag expansion
2536			# then just use the version in the new release.
2537			if samef merge/old/${F} merge/${OLDRELNUM}/${F}; then
2538				cp merge/${RELNUM}/${F} merge/new/${F}
2539				continue
2540			fi
2541
2542			cat <<-EOF
2543
2544The following file could not be merged automatically: ${F}
2545Press Enter to edit this file in ${EDITOR} and resolve the conflicts
2546manually...
2547			EOF
2548			while true; do
2549				read dummy </dev/tty
2550				${EDITOR} `pwd`/merge/new/${F} < /dev/tty
2551
2552				if ! grep -qE '^(<<<<<<<|=======|>>>>>>>)([[:space:]].*)?$' $(pwd)/merge/new/${F} ; then
2553					break
2554				fi
2555				cat <<-EOF
2556
2557Merge conflict markers remain in: ${F}
2558These must be resolved for the system to be functional.
2559
2560Press Enter to return to editing this file.
2561				EOF
2562			done
2563		done < failed.merges
2564		rm failed.merges
2565
2566		# Ask the user to confirm that he likes how the result
2567		# of merging files.
2568		while read F; do
2569			# Skip files which haven't changed except possibly
2570			# in their RCS tags.
2571			if [ -f merge/old/${F} ] && [ -f merge/new/${F} ] &&
2572			    samef merge/old/${F} merge/new/${F}; then
2573				continue
2574			fi
2575
2576			# Skip files where the installed file differs from
2577			# the old file only due to RCS tags.
2578			if [ -f merge/old/${F} ] &&
2579			    [ -f merge/${OLDRELNUM}/${F} ] &&
2580			    samef merge/old/${F} merge/${OLDRELNUM}/${F}; then
2581				continue
2582			fi
2583
2584			# Warn about files which are ceasing to exist.
2585			if ! [ -f merge/new/${F} ]; then
2586				cat <<-EOF
2587
2588The following file will be removed, as it no longer exists in
2589FreeBSD ${RELNUM}: ${F}
2590				EOF
2591				continuep < /dev/tty || return 1
2592				continue
2593			fi
2594
2595			# Print changes for the user's approval.
2596			cat <<-EOF
2597
2598The following changes, which occurred between FreeBSD ${OLDRELNUM} and
2599FreeBSD ${RELNUM} have been merged into ${F}:
2600EOF
2601			diff -U 5 -L "current version" -L "new version"	\
2602			    merge/old/${F} merge/new/${F} || true
2603			continuep < /dev/tty || return 1
2604		done < $1-paths
2605
2606		# Store merged files.
2607		while read F; do
2608			if [ -f merge/new/${F} ]; then
2609				V=`${SHA256} -q merge/new/${F}`
2610
2611				gzip -c < merge/new/${F} > files/${V}.gz
2612				echo "${F}|${V}"
2613			fi
2614		done < $1-paths > newhashes
2615
2616		# Pull lines out from $3 which need to be updated to
2617		# reflect merged files.
2618		while read F; do
2619			look "${F}|" $3
2620		done < $1-paths > $3-oldlines
2621
2622		# Update lines to reflect merged files
2623		join -t '|' -o 1.1,1.2,1.3,1.4,1.5,1.6,2.2,1.8		\
2624		    $3-oldlines newhashes > $3-newlines
2625
2626		# Remove old lines from $3 and add new lines.
2627		sort $3-oldlines |
2628		    comm -13 - $3 |
2629		    sort - $3-newlines > $3.tmp
2630		mv $3.tmp $3
2631
2632		# Clean up
2633		rm $1-paths newhashes $3-oldlines $3-newlines
2634		rm -rf merge/
2635	fi
2636
2637	# We're done with merging files.
2638	rm $1
2639}
2640
2641# Do the work involved in fetching upgrades to a new release
2642upgrade_run () {
2643	workdir_init || return 1
2644
2645	# Prepare the mirror list.
2646	fetch_pick_server_init && fetch_pick_server
2647
2648	# Try to fetch the public key until we run out of servers.
2649	while ! fetch_key; do
2650		fetch_pick_server || return 1
2651	done
2652
2653	# Try to fetch the metadata index signature ("tag") until we run
2654	# out of available servers; and sanity check the downloaded tag.
2655	while ! fetch_tag; do
2656		fetch_pick_server || return 1
2657	done
2658	fetch_tagsanity || return 1
2659
2660	# Fetch the INDEX-OLD and INDEX-ALL.
2661	fetch_metadata INDEX-OLD INDEX-ALL || return 1
2662
2663	# If StrictComponents is not "yes", generate a new components list
2664	# with only the components which appear to be installed.
2665	upgrade_guess_components INDEX-ALL || return 1
2666
2667	# Generate filtered INDEX-OLD and INDEX-ALL files containing only
2668	# the components we want and without anything marked as "Ignore".
2669	fetch_filter_metadata INDEX-OLD || return 1
2670	fetch_filter_metadata INDEX-ALL || return 1
2671
2672	# Merge the INDEX-OLD and INDEX-ALL files into INDEX-OLD.
2673	sort INDEX-OLD INDEX-ALL > INDEX-OLD.tmp
2674	mv INDEX-OLD.tmp INDEX-OLD
2675	rm INDEX-ALL
2676
2677	# Adjust variables for fetching files from the new release.
2678	OLDRELNUM=${RELNUM}
2679	RELNUM=${TARGETRELEASE}
2680	OLDFETCHDIR=${FETCHDIR}
2681	FETCHDIR=${RELNUM}/${ARCH}
2682
2683	# Try to fetch the NEW metadata index signature ("tag") until we run
2684	# out of available servers; and sanity check the downloaded tag.
2685	while ! fetch_tag; do
2686		fetch_pick_server || return 1
2687	done
2688
2689	# Fetch the new INDEX-ALL.
2690	fetch_metadata INDEX-ALL || return 1
2691
2692	# If StrictComponents is not "yes", COMPONENTS contains an entry
2693	# corresponding to the currently running kernel, and said kernel
2694	# does not exist in the new release, add "kernel/generic" to the
2695	# list of components.
2696	upgrade_guess_new_kernel INDEX-ALL || return 1
2697
2698	# Filter INDEX-ALL to contain only the components we want and without
2699	# anything marked as "Ignore".
2700	fetch_filter_metadata INDEX-ALL || return 1
2701
2702	# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
2703	# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
2704	upgrade_oldall_to_oldnew INDEX-OLD INDEX-ALL INDEX-NEW
2705
2706	# Translate /boot/${KERNCONF} or /boot/${NKERNCONF} into ${KERNELDIR}
2707	fetch_filter_kernel_names INDEX-NEW ${NKERNCONF}
2708	fetch_filter_kernel_names INDEX-OLD ${KERNCONF}
2709
2710	# For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
2711	# system and generate an INDEX-PRESENT file.
2712	fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2713
2714	# Based on ${MERGECHANGES}, generate a file tomerge-old with the
2715	# paths and hashes of old versions of files to merge.
2716	fetch_filter_mergechanges INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old
2717
2718	# Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
2719	# correspond to lines in INDEX-PRESENT with hashes not appearing
2720	# in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
2721	# INDEX-PRESENT has type - and there isn't a corresponding entry in
2722	# INDEX-OLD with type -.
2723	fetch_filter_unmodified_notpresent	\
2724	    INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old
2725
2726	# For each entry in INDEX-PRESENT of type -, remove any corresponding
2727	# entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
2728	# of type - from INDEX-PRESENT.
2729	fetch_filter_allowadd INDEX-PRESENT INDEX-NEW
2730
2731	# If ${ALLOWDELETE} != "yes", then remove any entries from
2732	# INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
2733	fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW
2734
2735	# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
2736	# INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
2737	# replace the corresponding line of INDEX-NEW with one having the
2738	# same metadata as the entry in INDEX-PRESENT.
2739	fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW
2740
2741	# Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
2742	# no need to update a file if it isn't changing.
2743	fetch_filter_uptodate INDEX-PRESENT INDEX-NEW
2744
2745	# Fetch "clean" files from the old release for merging changes.
2746	fetch_files_premerge tomerge-old
2747
2748	# Prepare to fetch files: Generate a list of the files we need,
2749	# copy the unmodified files we have into /files/, and generate
2750	# a list of patches to download.
2751	fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2752
2753	# Fetch patches from to-${RELNUM}/${ARCH}/bp/
2754	PATCHDIR=to-${RELNUM}/${ARCH}/bp
2755	fetch_files || return 1
2756
2757	# Merge configuration file changes.
2758	upgrade_merge tomerge-old INDEX-PRESENT INDEX-NEW || return 1
2759
2760	# Create and populate install manifest directory; and report what
2761	# updates are available.
2762	fetch_create_manifest || return 1
2763
2764	# Leave a note behind to tell the "install" command that the kernel
2765	# needs to be installed before the world.
2766	touch ${BDHASH}-install/kernelfirst
2767
2768	# Remind the user that they need to run "freebsd-update install"
2769	# to install the downloaded bits, in case they didn't RTFM.
2770	echo "To install the downloaded upgrades, run \"$0 install\"."
2771}
2772
2773# Make sure that all the file hashes mentioned in $@ have corresponding
2774# gzipped files stored in /files/.
2775install_verify () {
2776	# Generate a list of hashes
2777	cat $@ |
2778	    cut -f 2,7 -d '|' |
2779	    grep -E '^f' |
2780	    cut -f 2 -d '|' |
2781	    sort -u > filelist
2782
2783	# Make sure all the hashes exist
2784	while read HASH; do
2785		if ! [ -f files/${HASH}.gz ]; then
2786			echo -n "Update files missing -- "
2787			echo "this should never happen."
2788			echo "Re-run '$0 fetch'."
2789			return 1
2790		fi
2791	done < filelist
2792
2793	# Clean up
2794	rm filelist
2795}
2796
2797# Remove the system immutable flag from files
2798install_unschg () {
2799	# Generate file list
2800	cat $@ |
2801	    cut -f 1 -d '|' > filelist
2802
2803	# Remove flags
2804	while read F; do
2805		if ! [ -e ${BASEDIR}/${F} ]; then
2806			continue
2807		else
2808			echo ${BASEDIR}/${F}
2809		fi
2810	done < filelist | xargs chflags noschg || return 1
2811
2812	# Clean up
2813	rm filelist
2814}
2815
2816# Decide which directory name to use for kernel backups.
2817backup_kernel_finddir () {
2818	CNT=0
2819	while true ; do
2820		# Pathname does not exist, so it is OK use that name
2821		# for backup directory.
2822		if [ ! -e $BASEDIR/$BACKUPKERNELDIR ]; then
2823			return 0
2824		fi
2825
2826		# If directory do exist, we only use if it has our
2827		# marker file.
2828		if [ -d $BASEDIR/$BACKUPKERNELDIR -a \
2829			-e $BASEDIR/$BACKUPKERNELDIR/.freebsd-update ]; then
2830			return 0
2831		fi
2832
2833		# We could not use current directory name, so add counter to
2834		# the end and try again.
2835		CNT=$((CNT + 1))
2836		if [ $CNT -gt 9 ]; then
2837			echo "Could not find valid backup dir ($BASEDIR/$BACKUPKERNELDIR)"
2838			exit 1
2839		fi
2840		BACKUPKERNELDIR="`echo $BACKUPKERNELDIR | sed -Ee 's/[0-9]\$//'`"
2841		BACKUPKERNELDIR="${BACKUPKERNELDIR}${CNT}"
2842	done
2843}
2844
2845# Backup the current kernel using hardlinks, if not disabled by user.
2846# Since we delete all files in the directory used for previous backups
2847# we create a marker file called ".freebsd-update" in the directory so
2848# we can determine on the next run that the directory was created by
2849# freebsd-update and we then do not accidentally remove user files in
2850# the unlikely case that the user has created a directory with a
2851# conflicting name.
2852backup_kernel () {
2853	# Only make kernel backup is so configured.
2854	if [ $BACKUPKERNEL != yes ]; then
2855		return 0
2856	fi
2857
2858	# Decide which directory name to use for kernel backups.
2859	backup_kernel_finddir
2860
2861	# Remove old kernel backup files.  If $BACKUPKERNELDIR was
2862	# "not ours", backup_kernel_finddir would have exited, so
2863	# deleting the directory content is as safe as we can make it.
2864	if [ -d $BASEDIR/$BACKUPKERNELDIR ]; then
2865		rm -fr $BASEDIR/$BACKUPKERNELDIR
2866	fi
2867
2868	# Create directories for backup.
2869	mkdir -p $BASEDIR/$BACKUPKERNELDIR
2870	mtree -cdn -p "${BASEDIR}/${KERNELDIR}" | \
2871	    mtree -Ue -p "${BASEDIR}/${BACKUPKERNELDIR}" > /dev/null
2872
2873	# Mark the directory as having been created by freebsd-update.
2874	touch $BASEDIR/$BACKUPKERNELDIR/.freebsd-update
2875	if [ $? -ne 0 ]; then
2876		echo "Could not create kernel backup directory"
2877		exit 1
2878	fi
2879
2880	# Disable pathname expansion to be sure *.symbols is not
2881	# expanded.
2882	set -f
2883
2884	# Use find to ignore symbol files, unless disabled by user.
2885	if [ $BACKUPKERNELSYMBOLFILES = yes ]; then
2886		FINDFILTER=""
2887	else
2888		FINDFILTER="-a ! -name *.debug -a ! -name *.symbols"
2889	fi
2890
2891	# Backup all the kernel files using hardlinks.
2892	(cd ${BASEDIR}/${KERNELDIR} && find . -type f $FINDFILTER -exec \
2893	    cp -pl '{}' ${BASEDIR}/${BACKUPKERNELDIR}/'{}' \;)
2894
2895	# Re-enable patchname expansion.
2896	set +f
2897}
2898
2899# Install new files
2900install_from_index () {
2901	# First pass: Do everything apart from setting file flags.  We
2902	# can't set flags yet, because schg inhibits hard linking.
2903	sort -k 1,1 -t '|' $1 |
2904	    tr '|' ' ' |
2905	    while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
2906		case ${TYPE} in
2907		d)
2908			# Create a directory
2909			install -d -o ${OWNER} -g ${GROUP}		\
2910			    -m ${PERM} ${BASEDIR}/${FPATH}
2911			;;
2912		f)
2913			if [ -z "${LINK}" ]; then
2914				# Create a file, without setting flags.
2915				gunzip < files/${HASH}.gz > ${HASH}
2916				install -S -o ${OWNER} -g ${GROUP}	\
2917				    -m ${PERM} ${HASH} ${BASEDIR}/${FPATH}
2918				rm ${HASH}
2919			else
2920				# Create a hard link.
2921				ln -f ${BASEDIR}/${LINK} ${BASEDIR}/${FPATH}
2922			fi
2923			;;
2924		L)
2925			# Create a symlink
2926			ln -sfh ${HASH} ${BASEDIR}/${FPATH}
2927			;;
2928		esac
2929	    done
2930
2931	# Perform a second pass, adding file flags.
2932	tr '|' ' ' < $1 |
2933	    while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
2934		if [ ${TYPE} = "f" ] &&
2935		    ! [ ${FLAGS} = "0" ]; then
2936			chflags ${FLAGS} ${BASEDIR}/${FPATH}
2937		fi
2938	    done
2939}
2940
2941# Remove files which we want to delete
2942install_delete () {
2943	# Generate list of new files
2944	cut -f 1 -d '|' < $2 |
2945	    sort > newfiles
2946
2947	# Generate subindex of old files we want to nuke
2948	sort -k 1,1 -t '|' $1 |
2949	    join -t '|' -v 1 - newfiles |
2950	    sort -r -k 1,1 -t '|' |
2951	    cut -f 1,2 -d '|' |
2952	    tr '|' ' ' > killfiles
2953
2954	# Remove the offending bits
2955	while read FPATH TYPE; do
2956		case ${TYPE} in
2957		d)
2958			rmdir ${BASEDIR}/${FPATH}
2959			;;
2960		f)
2961			rm ${BASEDIR}/${FPATH}
2962			;;
2963		L)
2964			rm ${BASEDIR}/${FPATH}
2965			;;
2966		esac
2967	done < killfiles
2968
2969	# Clean up
2970	rm newfiles killfiles
2971}
2972
2973# Install new files, delete old files, and update generated files
2974install_files () {
2975	# If we haven't already dealt with the kernel, deal with it.
2976	if ! [ -f $1/kerneldone ]; then
2977		grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
2978		grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
2979
2980		# Backup current kernel before installing a new one
2981		backup_kernel || return 1
2982
2983		# Install new files
2984		install_from_index INDEX-NEW || return 1
2985
2986		# Remove files which need to be deleted
2987		install_delete INDEX-OLD INDEX-NEW || return 1
2988
2989		# Update linker.hints if necessary
2990		if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
2991			kldxref -R ${BASEDIR}/boot/ 2>/dev/null
2992		fi
2993
2994		# We've finished updating the kernel.
2995		touch $1/kerneldone
2996
2997		# Do we need to ask for a reboot now?
2998		if [ -f $1/kernelfirst ] &&
2999		    [ -s INDEX-OLD -o -s INDEX-NEW ]; then
3000			cat <<-EOF
3001
3002Kernel updates have been installed.  Please reboot and run
3003"$0 install" again to finish installing updates.
3004			EOF
3005			exit 0
3006		fi
3007	fi
3008
3009	# If we haven't already dealt with the world, deal with it.
3010	if ! [ -f $1/worlddone ]; then
3011		# Create any necessary directories first
3012		grep -vE '^/boot/' $1/INDEX-NEW |
3013		    grep -E '^[^|]+\|d\|' > INDEX-NEW
3014		install_from_index INDEX-NEW || return 1
3015
3016		# Install new runtime linker
3017		grep -vE '^/boot/' $1/INDEX-NEW |
3018		    grep -vE '^[^|]+\|d\|' |
3019		    grep -E '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3020		install_from_index INDEX-NEW || return 1
3021
3022		# Install new shared libraries next
3023		grep -vE '^/boot/' $1/INDEX-NEW |
3024		    grep -vE '^[^|]+\|d\|' |
3025		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3026		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3027		install_from_index INDEX-NEW || return 1
3028
3029		# Deal with everything else
3030		grep -vE '^/boot/' $1/INDEX-OLD |
3031		    grep -vE '^[^|]+\|d\|' |
3032		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3033		    grep -vE '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-OLD
3034		grep -vE '^/boot/' $1/INDEX-NEW |
3035		    grep -vE '^[^|]+\|d\|' |
3036		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3037		    grep -vE '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3038		install_from_index INDEX-NEW || return 1
3039		install_delete INDEX-OLD INDEX-NEW || return 1
3040
3041		# Restart sshd if running (PR263489).  Note that this does not
3042		# affect child sshd processes handling existing sessions.
3043		if service sshd status >/dev/null 2>/dev/null; then
3044			echo
3045			echo "Restarting sshd after upgrade"
3046			service sshd restart
3047		fi
3048
3049		# Rehash certs if we actually have certctl installed.
3050		if which certctl>/dev/null; then
3051			env DESTDIR=${BASEDIR} certctl rehash
3052		fi
3053
3054		# Rebuild generated pwd files and /etc/login.conf.db.
3055		pwd_mkdb -d ${BASEDIR}/etc -p ${BASEDIR}/etc/master.passwd
3056		cap_mkdb ${BASEDIR}/etc/login.conf
3057
3058		# Rebuild man page databases, if necessary.
3059		for D in /usr/share/man /usr/share/openssl/man; do
3060			if [ ! -d ${BASEDIR}/$D ]; then
3061				continue
3062			fi
3063			if [ -f ${BASEDIR}/$D/mandoc.db ] && \
3064			    [ -z "$(find ${BASEDIR}/$D -type f -newer ${BASEDIR}/$D/mandoc.db)" ]; then
3065				continue;
3066			fi
3067			makewhatis ${BASEDIR}/$D
3068		done
3069
3070		# We've finished installing the world and deleting old files
3071		# which are not shared libraries.
3072		touch $1/worlddone
3073
3074		# Do we need to ask the user to portupgrade now?
3075		grep -vE '^/boot/' $1/INDEX-NEW |
3076		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' |
3077		    cut -f 1 -d '|' |
3078		    sort > newfiles
3079		if grep -vE '^/boot/' $1/INDEX-OLD |
3080		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' |
3081		    cut -f 1 -d '|' |
3082		    sort |
3083		    join -v 1 - newfiles |
3084		    grep -q .; then
3085			cat <<-EOF
3086
3087Completing this upgrade requires removing old shared object files.
3088Please rebuild all installed 3rd party software (e.g., programs
3089installed from the ports tree) and then run "$0 install"
3090again to finish installing updates.
3091			EOF
3092			rm newfiles
3093			exit 0
3094		fi
3095		rm newfiles
3096	fi
3097
3098	# Remove old shared libraries
3099	grep -vE '^/boot/' $1/INDEX-NEW |
3100	    grep -vE '^[^|]+\|d\|' |
3101	    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3102	grep -vE '^/boot/' $1/INDEX-OLD |
3103	    grep -vE '^[^|]+\|d\|' |
3104	    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-OLD
3105	install_delete INDEX-OLD INDEX-NEW || return 1
3106
3107	# Remove old directories
3108	grep -vE '^/boot/' $1/INDEX-NEW |
3109	    grep -E '^[^|]+\|d\|' > INDEX-NEW
3110	grep -vE '^/boot/' $1/INDEX-OLD |
3111	    grep -E '^[^|]+\|d\|' > INDEX-OLD
3112	install_delete INDEX-OLD INDEX-NEW || return 1
3113
3114	# Remove temporary files
3115	rm INDEX-OLD INDEX-NEW
3116}
3117
3118# Rearrange bits to allow the installed updates to be rolled back
3119install_setup_rollback () {
3120	# Remove the "reboot after installing kernel", "kernel updated", and
3121	# "finished installing the world" flags if present -- they are
3122	# irrelevant when rolling back updates.
3123	if [ -f ${BDHASH}-install/kernelfirst ]; then
3124		rm ${BDHASH}-install/kernelfirst
3125		rm ${BDHASH}-install/kerneldone
3126	fi
3127	if [ -f ${BDHASH}-install/worlddone ]; then
3128		rm ${BDHASH}-install/worlddone
3129	fi
3130
3131	if [ -L ${BDHASH}-rollback ]; then
3132		mv ${BDHASH}-rollback ${BDHASH}-install/rollback
3133	fi
3134
3135	mv ${BDHASH}-install ${BDHASH}-rollback
3136}
3137
3138# Actually install updates
3139install_run () {
3140	echo -n "Installing updates..."
3141
3142	# Make sure we have all the files we should have
3143	install_verify ${BDHASH}-install/INDEX-OLD	\
3144	    ${BDHASH}-install/INDEX-NEW || return 1
3145
3146	# Remove system immutable flag from files
3147	install_unschg ${BDHASH}-install/INDEX-OLD	\
3148	    ${BDHASH}-install/INDEX-NEW || return 1
3149
3150	# Install new files, delete old files, and update linker.hints
3151	install_files ${BDHASH}-install || return 1
3152
3153	# Rearrange bits to allow the installed updates to be rolled back
3154	install_setup_rollback
3155
3156	echo " done."
3157}
3158
3159# Rearrange bits to allow the previous set of updates to be rolled back next.
3160rollback_setup_rollback () {
3161	if [ -L ${BDHASH}-rollback/rollback ]; then
3162		mv ${BDHASH}-rollback/rollback rollback-tmp
3163		rm -r ${BDHASH}-rollback/
3164		rm ${BDHASH}-rollback
3165		mv rollback-tmp ${BDHASH}-rollback
3166	else
3167		rm -r ${BDHASH}-rollback/
3168		rm ${BDHASH}-rollback
3169	fi
3170}
3171
3172# Install old files, delete new files, and update linker.hints
3173rollback_files () {
3174	# Install old shared library files which don't have the same path as
3175	# a new shared library file.
3176	grep -vE '^/boot/' $1/INDEX-NEW |
3177	    grep -E '/lib/.*\.so\.[0-9]+\|' |
3178	    cut -f 1 -d '|' |
3179	    sort > INDEX-NEW.libs.flist
3180	grep -vE '^/boot/' $1/INDEX-OLD |
3181	    grep -E '/lib/.*\.so\.[0-9]+\|' |
3182	    sort -k 1,1 -t '|' - |
3183	    join -t '|' -v 1 - INDEX-NEW.libs.flist > INDEX-OLD
3184	install_from_index INDEX-OLD || return 1
3185
3186	# Deal with files which are neither kernel nor shared library
3187	grep -vE '^/boot/' $1/INDEX-OLD |
3188	    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
3189	grep -vE '^/boot/' $1/INDEX-NEW |
3190	    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
3191	install_from_index INDEX-OLD || return 1
3192	install_delete INDEX-NEW INDEX-OLD || return 1
3193
3194	# Install any old shared library files which we didn't install above.
3195	grep -vE '^/boot/' $1/INDEX-OLD |
3196	    grep -E '/lib/.*\.so\.[0-9]+\|' |
3197	    sort -k 1,1 -t '|' - |
3198	    join -t '|' - INDEX-NEW.libs.flist > INDEX-OLD
3199	install_from_index INDEX-OLD || return 1
3200
3201	# Delete unneeded shared library files
3202	grep -vE '^/boot/' $1/INDEX-OLD |
3203	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
3204	grep -vE '^/boot/' $1/INDEX-NEW |
3205	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
3206	install_delete INDEX-NEW INDEX-OLD || return 1
3207
3208	# Deal with kernel files
3209	grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
3210	grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
3211	install_from_index INDEX-OLD || return 1
3212	install_delete INDEX-NEW INDEX-OLD || return 1
3213	if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
3214		kldxref -R /boot/ 2>/dev/null
3215	fi
3216
3217	# Remove temporary files
3218	rm INDEX-OLD INDEX-NEW INDEX-NEW.libs.flist
3219}
3220
3221# Actually rollback updates
3222rollback_run () {
3223	echo -n "Uninstalling updates..."
3224
3225	# If there are updates waiting to be installed, remove them; we
3226	# want the user to re-run 'fetch' after rolling back updates.
3227	if [ -L ${BDHASH}-install ]; then
3228		rm -r ${BDHASH}-install/
3229		rm ${BDHASH}-install
3230	fi
3231
3232	# Make sure we have all the files we should have
3233	install_verify ${BDHASH}-rollback/INDEX-NEW	\
3234	    ${BDHASH}-rollback/INDEX-OLD || return 1
3235
3236	# Remove system immutable flag from files
3237	install_unschg ${BDHASH}-rollback/INDEX-NEW	\
3238	    ${BDHASH}-rollback/INDEX-OLD || return 1
3239
3240	# Install old files, delete new files, and update linker.hints
3241	rollback_files ${BDHASH}-rollback || return 1
3242
3243	# Remove the rollback directory and the symlink pointing to it; and
3244	# rearrange bits to allow the previous set of updates to be rolled
3245	# back next.
3246	rollback_setup_rollback
3247
3248	echo " done."
3249}
3250
3251# Compare INDEX-ALL and INDEX-PRESENT and print warnings about differences.
3252IDS_compare () {
3253	# Get all the lines which mismatch in something other than file
3254	# flags.  We ignore file flags because sysinstall doesn't seem to
3255	# set them when it installs FreeBSD; warning about these adds a
3256	# very large amount of noise.
3257	cut -f 1-5,7-8 -d '|' $1 > $1.noflags
3258	sort -k 1,1 -t '|' $1.noflags > $1.sorted
3259	cut -f 1-5,7-8 -d '|' $2 |
3260	    comm -13 $1.noflags - |
3261	    fgrep -v '|-|||||' |
3262	    sort -k 1,1 -t '|' |
3263	    join -t '|' $1.sorted - > INDEX-NOTMATCHING
3264
3265	# Ignore files which match IDSIGNOREPATHS.
3266	for X in ${IDSIGNOREPATHS}; do
3267		grep -E "^${X}" INDEX-NOTMATCHING
3268	done |
3269	    sort -u |
3270	    comm -13 - INDEX-NOTMATCHING > INDEX-NOTMATCHING.tmp
3271	mv INDEX-NOTMATCHING.tmp INDEX-NOTMATCHING
3272
3273	# Go through the lines and print warnings.
3274	local IFS='|'
3275	while read FPATH TYPE OWNER GROUP PERM HASH LINK P_TYPE P_OWNER P_GROUP P_PERM P_HASH P_LINK; do
3276		# Warn about different object types.
3277		if ! [ "${TYPE}" = "${P_TYPE}" ]; then
3278			echo -n "${FPATH} is a "
3279			case "${P_TYPE}" in
3280			f)	echo -n "regular file, "
3281				;;
3282			d)	echo -n "directory, "
3283				;;
3284			L)	echo -n "symlink, "
3285				;;
3286			esac
3287			echo -n "but should be a "
3288			case "${TYPE}" in
3289			f)	echo -n "regular file."
3290				;;
3291			d)	echo -n "directory."
3292				;;
3293			L)	echo -n "symlink."
3294				;;
3295			esac
3296			echo
3297
3298			# Skip other tests, since they don't make sense if
3299			# we're comparing different object types.
3300			continue
3301		fi
3302
3303		# Warn about different owners.
3304		if ! [ "${OWNER}" = "${P_OWNER}" ]; then
3305			echo -n "${FPATH} is owned by user id ${P_OWNER}, "
3306			echo "but should be owned by user id ${OWNER}."
3307		fi
3308
3309		# Warn about different groups.
3310		if ! [ "${GROUP}" = "${P_GROUP}" ]; then
3311			echo -n "${FPATH} is owned by group id ${P_GROUP}, "
3312			echo "but should be owned by group id ${GROUP}."
3313		fi
3314
3315		# Warn about different permissions.  We do not warn about
3316		# different permissions on symlinks, since some archivers
3317		# don't extract symlink permissions correctly and they are
3318		# ignored anyway.
3319		if ! [ "${PERM}" = "${P_PERM}" ] &&
3320		    ! [ "${TYPE}" = "L" ]; then
3321			echo -n "${FPATH} has ${P_PERM} permissions, "
3322			echo "but should have ${PERM} permissions."
3323		fi
3324
3325		# Warn about different file hashes / symlink destinations.
3326		if ! [ "${HASH}" = "${P_HASH}" ]; then
3327			if [ "${TYPE}" = "L" ]; then
3328				echo -n "${FPATH} is a symlink to ${P_HASH}, "
3329				echo "but should be a symlink to ${HASH}."
3330			fi
3331			if [ "${TYPE}" = "f" ]; then
3332				echo -n "${FPATH} has SHA256 hash ${P_HASH}, "
3333				echo "but should have SHA256 hash ${HASH}."
3334			fi
3335		fi
3336
3337		# We don't warn about different hard links, since some
3338		# some archivers break hard links, and as long as the
3339		# underlying data is correct they really don't matter.
3340	done < INDEX-NOTMATCHING
3341
3342	# Clean up
3343	rm $1 $1.noflags $1.sorted $2 INDEX-NOTMATCHING
3344}
3345
3346# Do the work involved in comparing the system to a "known good" index
3347IDS_run () {
3348	workdir_init || return 1
3349
3350	# Prepare the mirror list.
3351	fetch_pick_server_init && fetch_pick_server
3352
3353	# Try to fetch the public key until we run out of servers.
3354	while ! fetch_key; do
3355		fetch_pick_server || return 1
3356	done
3357
3358	# Try to fetch the metadata index signature ("tag") until we run
3359	# out of available servers; and sanity check the downloaded tag.
3360	while ! fetch_tag; do
3361		fetch_pick_server || return 1
3362	done
3363	fetch_tagsanity || return 1
3364
3365	# Fetch INDEX-OLD and INDEX-ALL.
3366	fetch_metadata INDEX-OLD INDEX-ALL || return 1
3367
3368	# Generate filtered INDEX-OLD and INDEX-ALL files containing only
3369	# the components we want and without anything marked as "Ignore".
3370	fetch_filter_metadata INDEX-OLD || return 1
3371	fetch_filter_metadata INDEX-ALL || return 1
3372
3373	# Merge the INDEX-OLD and INDEX-ALL files into INDEX-ALL.
3374	sort INDEX-OLD INDEX-ALL > INDEX-ALL.tmp
3375	mv INDEX-ALL.tmp INDEX-ALL
3376	rm INDEX-OLD
3377
3378	# Translate /boot/${KERNCONF} to ${KERNELDIR}
3379	fetch_filter_kernel_names INDEX-ALL ${KERNCONF}
3380
3381	# Inspect the system and generate an INDEX-PRESENT file.
3382	fetch_inspect_system INDEX-ALL INDEX-PRESENT /dev/null || return 1
3383
3384	# Compare INDEX-ALL and INDEX-PRESENT and print warnings about any
3385	# differences.
3386	IDS_compare INDEX-ALL INDEX-PRESENT
3387}
3388
3389#### Main functions -- call parameter-handling and core functions
3390
3391# Using the command line, configuration file, and defaults,
3392# set all the parameters which are needed later.
3393get_params () {
3394	init_params
3395	parse_cmdline $@
3396	parse_conffile
3397	default_params
3398}
3399
3400# Fetch command.  Make sure that we're being called
3401# interactively, then run fetch_check_params and fetch_run
3402cmd_fetch () {
3403	finalize_components_config ${COMPONENTS}
3404	if [ ! -t 0 -a $NOTTYOK -eq 0 ]; then
3405		echo -n "`basename $0` fetch should not "
3406		echo "be run non-interactively."
3407		echo "Run `basename $0` cron instead."
3408		exit 1
3409	fi
3410	fetch_check_params
3411	fetch_run || exit 1
3412	ISFETCHED=1
3413}
3414
3415# Cron command.  Make sure the parameters are sensible; wait
3416# rand(3600) seconds; then fetch updates.  While fetching updates,
3417# send output to a temporary file; only print that file if the
3418# fetching failed.
3419cmd_cron () {
3420	fetch_check_params
3421	sleep `jot -r 1 0 3600`
3422
3423	TMPFILE=`mktemp /tmp/freebsd-update.XXXXXX` || exit 1
3424	finalize_components_config ${COMPONENTS} >> ${TMPFILE}
3425	if ! fetch_run >> ${TMPFILE} ||
3426	    ! grep -q "No updates needed" ${TMPFILE} ||
3427	    [ ${VERBOSELEVEL} = "debug" ]; then
3428		mail -s "`hostname` security updates" ${MAILTO} < ${TMPFILE}
3429	fi
3430
3431	rm ${TMPFILE}
3432}
3433
3434# Fetch files for upgrading to a new release.
3435cmd_upgrade () {
3436	finalize_components_config ${COMPONENTS}
3437	upgrade_check_params
3438	upgrade_run || exit 1
3439}
3440
3441# Check if there are fetched updates ready to install.
3442# Chdir into the working directory.
3443cmd_updatesready () {
3444	finalize_components_config ${COMPONENTS}
3445	# Check if working directory exists (if not, no updates pending)
3446	if ! [ -e "${WORKDIR}" ]; then
3447		echo "No updates are available to install."
3448		exit 2
3449	fi
3450
3451	# Change into working directory (fail if no permission/directory etc.)
3452	cd ${WORKDIR} || exit 1
3453
3454	# Construct a unique name from ${BASEDIR}
3455	BDHASH=`echo ${BASEDIR} | sha256 -q`
3456
3457	# Check that we have updates ready to install
3458	if ! [ -L ${BDHASH}-install ]; then
3459		echo "No updates are available to install."
3460		exit 2
3461	fi
3462
3463	echo "There are updates available to install."
3464	echo "Run '$0 install' to proceed."
3465}
3466
3467# Install downloaded updates.
3468cmd_install () {
3469	finalize_components_config ${COMPONENTS}
3470	install_check_params
3471	install_create_be
3472	install_run || exit 1
3473}
3474
3475# Rollback most recently installed updates.
3476cmd_rollback () {
3477	finalize_components_config ${COMPONENTS}
3478	rollback_check_params
3479	rollback_run || exit 1
3480}
3481
3482# Compare system against a "known good" index.
3483cmd_IDS () {
3484	finalize_components_config ${COMPONENTS}
3485	IDS_check_params
3486	IDS_run || exit 1
3487}
3488
3489# Output configuration.
3490cmd_showconfig () {
3491	finalize_components_config ${COMPONENTS}
3492	for X in ${CONFIGOPTIONS}; do
3493		echo $X=$(eval echo \$${X})
3494	done
3495}
3496
3497#### Entry point
3498
3499# Make sure we find utilities from the base system
3500export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH}
3501
3502# Set a pager if the user doesn't
3503if [ -z "$PAGER" ]; then
3504	PAGER=/usr/bin/less
3505fi
3506
3507# Set LC_ALL in order to avoid problems with character ranges like [A-Z].
3508export LC_ALL=C
3509
3510# Clear environment variables that may affect operation of tools that we use.
3511unset GREP_OPTIONS
3512
3513get_params $@
3514for COMMAND in ${COMMANDS}; do
3515	cmd_${COMMAND}
3516done
3517