xref: /freebsd/usr.sbin/freebsd-update/freebsd-update.sh (revision 2e3507c25e42292b45a5482e116d278f5515d04d)

#!/bin/sh

#-
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright 2004-2007 Colin Percival
# All rights reserved
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted providing that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

#### Usage function -- called from command-line handling code.

# Usage instructions.  Options not listed:
# --debug	-- don't filter output from utilities
# --no-stats	-- don't show progress statistics while fetching files
usage () {
	cat <<EOF
usage: `basename $0` [options] command ...

Options:
  -b basedir   -- Operate on a system mounted at basedir
                  (default: /)
  -d workdir   -- Store working files in workdir
                  (default: /var/db/freebsd-update/)
  -f conffile  -- Read configuration options from conffile
                  (default: /etc/freebsd-update.conf)
  -F           -- Force a fetch operation to proceed in the
                  case of an unfinished upgrade
  -j jail      -- Operate on the given jail specified by jid or name
  -k KEY       -- Trust an RSA key with SHA256 hash of KEY
  -r release   -- Target for upgrade (e.g., 13.2-RELEASE)
  -s server    -- Server from which to fetch updates
                  (default: update.FreeBSD.org)
  -t address   -- Mail output of cron command, if any, to address
                  (default: root)
  --not-running-from-cron
               -- Run without a tty, for use by automated tools
  --currently-running release
               -- Update as if currently running this release
Commands:
  fetch        -- Fetch updates from server
  cron         -- Sleep rand(3600) seconds, fetch updates, and send an
                  email if updates were found
  upgrade      -- Fetch upgrades to FreeBSD version specified via -r option
  updatesready -- Check if there are fetched updates ready to install
  install      -- Install downloaded updates or upgrades
  rollback     -- Uninstall most recently installed updates
  IDS          -- Compare the system against an index of "known good" files
  showconfig   -- Show configuration
EOF
	exit 0
}

#### Configuration processing functions

#-
# Configuration options are set in the following order of priority:
# 1. Command line options
# 2. Configuration file options
# 3. Default options
# In addition, certain options (e.g., IgnorePaths) can be specified multiple
# times and (as long as these are all in the same place, e.g., inside the
# configuration file) they will accumulate.  Finally, because the path to the
# configuration file can be specified at the command line, the entire command
# line must be processed before we start reading the configuration file.
#
# Sound like a mess?  It is.  Here's how we handle this:
# 1. Initialize CONFFILE and all the options to "".
# 2. Process the command line.  Throw an error if a non-accumulating option
#    is specified twice.
# 3. If CONFFILE is "", set CONFFILE to /etc/freebsd-update.conf .
# 4. For all the configuration options X, set X_saved to X.
# 5. Initialize all the options to "".
# 6. Read CONFFILE line by line, parsing options.
# 7. For each configuration option X, set X to X_saved iff X_saved is not "".
# 8. Repeat steps 4-7, except setting options to their default values at (6).

CONFIGOPTIONS="KEYPRINT WORKDIR SERVERNAME MAILTO ALLOWADD ALLOWDELETE
    KEEPMODIFIEDMETADATA COMPONENTS IGNOREPATHS UPDATEIFUNMODIFIED
    BASEDIR VERBOSELEVEL TARGETRELEASE STRICTCOMPONENTS MERGECHANGES
    IDSIGNOREPATHS BACKUPKERNEL BACKUPKERNELDIR BACKUPKERNELSYMBOLFILES"

# Set all the configuration options to "".
nullconfig () {
	for X in ${CONFIGOPTIONS}; do
		eval ${X}=""
	done
}

# For each configuration option X, set X_saved to X.
saveconfig () {
	for X in ${CONFIGOPTIONS}; do
		eval ${X}_saved=\$${X}
	done
}

# For each configuration option X, set X to X_saved if X_saved is not "".
mergeconfig () {
	for X in ${CONFIGOPTIONS}; do
		eval _=\$${X}_saved
		if ! [ -z "${_}" ]; then
			eval ${X}=\$${X}_saved
		fi
	done
}

# Set the trusted keyprint.
config_KeyPrint () {
	if [ -z ${KEYPRINT} ]; then
		KEYPRINT=$1
	else
		return 1
	fi
}

# Set the working directory.
config_WorkDir () {
	if [ -z ${WORKDIR} ]; then
		WORKDIR=$1
	else
		return 1
	fi
}

# Set the name of the server (pool) from which to fetch updates
config_ServerName () {
	if [ -z ${SERVERNAME} ]; then
		SERVERNAME=$1
	else
		return 1
	fi
}

# Set the address to which 'cron' output will be mailed.
config_MailTo () {
	if [ -z ${MAILTO} ]; then
		MAILTO=$1
	else
		return 1
	fi
}

# Set whether FreeBSD Update is allowed to add files (or directories, or
# symlinks) which did not previously exist.
config_AllowAdd () {
	if [ -z ${ALLOWADD} ]; then
		case $1 in
		[Yy][Ee][Ss])
			ALLOWADD=yes
			;;
		[Nn][Oo])
			ALLOWADD=no
			;;
		*)
			return 1
			;;
		esac
	else
		return 1
	fi
}

# Set whether FreeBSD Update is allowed to remove files/directories/symlinks.
config_AllowDelete () {
	if [ -z ${ALLOWDELETE} ]; then
		case $1 in
		[Yy][Ee][Ss])
			ALLOWDELETE=yes
			;;
		[Nn][Oo])
			ALLOWDELETE=no
			;;
		*)
			return 1
			;;
		esac
	else
		return 1
	fi
}

# Set whether FreeBSD Update should keep existing inode ownership,
# permissions, and flags, in the event that they have been modified locally
# after the release.
config_KeepModifiedMetadata () {
	if [ -z ${KEEPMODIFIEDMETADATA} ]; then
		case $1 in
		[Yy][Ee][Ss])
			KEEPMODIFIEDMETADATA=yes
			;;
		[Nn][Oo])
			KEEPMODIFIEDMETADATA=no
			;;
		*)
			return 1
			;;
		esac
	else
		return 1
	fi
}

# Add to the list of components which should be kept updated.
config_Components () {
	for C in $@; do
		COMPONENTS="${COMPONENTS} ${C}"
	done
}

# Remove src component from list if it isn't installed
finalize_components_config () {
	COMPONENTS=""
	for C in $@; do
		if [ "$C" = "src" ]; then
			if [ -e "${BASEDIR}/usr/src/COPYRIGHT" ]; then
				COMPONENTS="${COMPONENTS} ${C}"
			else
				echo "src component not installed, skipped"
			fi
		else
			COMPONENTS="${COMPONENTS} ${C}"
		fi
	done
}

# Add to the list of paths under which updates will be ignored.
config_IgnorePaths () {
	for C in $@; do
		IGNOREPATHS="${IGNOREPATHS} ${C}"
	done
}

# Add to the list of paths which IDS should ignore.
config_IDSIgnorePaths () {
	for C in $@; do
		IDSIGNOREPATHS="${IDSIGNOREPATHS} ${C}"
	done
}

# Add to the list of paths within which updates will be performed only if the
# file on disk has not been modified locally.
config_UpdateIfUnmodified () {
	for C in $@; do
		UPDATEIFUNMODIFIED="${UPDATEIFUNMODIFIED} ${C}"
	done
}

# Add to the list of paths within which updates to text files will be merged
# instead of overwritten.
config_MergeChanges () {
	for C in $@; do
		MERGECHANGES="${MERGECHANGES} ${C}"
	done
}

# Work on a FreeBSD installation mounted under $1
config_BaseDir () {
	if [ -z ${BASEDIR} ]; then
		BASEDIR=$1
	else
		return 1
	fi
}

# When fetching upgrades, should we assume the user wants exactly the
# components listed in COMPONENTS, rather than trying to guess based on
# what's currently installed?
config_StrictComponents () {
	if [ -z ${STRICTCOMPONENTS} ]; then
		case $1 in
		[Yy][Ee][Ss])
			STRICTCOMPONENTS=yes
			;;
		[Nn][Oo])
			STRICTCOMPONENTS=no
			;;
		*)
			return 1
			;;
		esac
	else
		return 1
	fi
}

# Upgrade to FreeBSD $1
config_TargetRelease () {
	if [ -z ${TARGETRELEASE} ]; then
		TARGETRELEASE=$1
	else
		return 1
	fi
	if echo ${TARGETRELEASE} | grep -qE '^[0-9.]+$'; then
		TARGETRELEASE="${TARGETRELEASE}-RELEASE"
	fi
}

# Pretend current release is FreeBSD $1
config_SourceRelease () {
	UNAME_r=$1
	if echo ${UNAME_r} | grep -qE '^[0-9.]+$'; then
		UNAME_r="${UNAME_r}-RELEASE"
	fi
	export UNAME_r
}

# Get the Jail's path and the version of its installed userland
config_TargetJail () {
	JAIL=$1
	UNAME_r=$(freebsd-version -j ${JAIL})
	BASEDIR=$(jls -j ${JAIL} -h path | awk 'NR == 2 {print}')
	if [ -z ${BASEDIR} ] || [ -z ${UNAME_r} ]; then
		echo "The specified jail either doesn't exist or" \
		      "does not have freebsd-version."
		exit 1
	fi
	export UNAME_r
}

# Define what happens to output of utilities
config_VerboseLevel () {
	if [ -z ${VERBOSELEVEL} ]; then
		case $1 in
		[Dd][Ee][Bb][Uu][Gg])
			VERBOSELEVEL=debug
			;;
		[Nn][Oo][Ss][Tt][Aa][Tt][Ss])
			VERBOSELEVEL=nostats
			;;
		[Ss][Tt][Aa][Tt][Ss])
			VERBOSELEVEL=stats
			;;
		*)
			return 1
			;;
		esac
	else
		return 1
	fi
}

config_BackupKernel () {
	if [ -z ${BACKUPKERNEL} ]; then
		case $1 in
		[Yy][Ee][Ss])
			BACKUPKERNEL=yes
			;;
		[Nn][Oo])
			BACKUPKERNEL=no
			;;
		*)
			return 1
			;;
		esac
	else
		return 1
	fi
}

config_BackupKernelDir () {
	if [ -z ${BACKUPKERNELDIR} ]; then
		if [ -z "$1" ]; then
			echo "BackupKernelDir set to empty dir"
			return 1
		fi

		# We check for some paths which would be extremely odd
		# to use, but which could cause a lot of problems if
		# used.
		case $1 in
		/|/bin|/boot|/etc|/lib|/libexec|/sbin|/usr|/var)
			echo "BackupKernelDir set to invalid path $1"
			return 1
			;;
		/*)
			BACKUPKERNELDIR=$1
			;;
		*)
			echo "BackupKernelDir ($1) is not an absolute path"
			return 1
			;;
		esac
	else
		return 1
	fi
}

config_BackupKernelSymbolFiles () {
	if [ -z ${BACKUPKERNELSYMBOLFILES} ]; then
		case $1 in
		[Yy][Ee][Ss])
			BACKUPKERNELSYMBOLFILES=yes
			;;
		[Nn][Oo])
			BACKUPKERNELSYMBOLFILES=no
			;;
		*)
			return 1
			;;
		esac
	else
		return 1
	fi
}

config_CreateBootEnv () {
	if [ -z ${BOOTENV} ]; then
		case $1 in
		[Yy][Ee][Ss])
			BOOTENV=yes
			;;
		[Nn][Oo])
			BOOTENV=no
			;;
		*)
			return 1
			;;
		esac
	else
		return 1
	fi
}
# Handle one line of configuration
configline () {
	if [ $# -eq 0 ]; then
		return
	fi

	OPT=$1
	shift
	config_${OPT} $@
}

#### Parameter handling functions.

# Initialize parameters to null, just in case they're
# set in the environment.
init_params () {
	# Configration settings
	nullconfig

	# No configuration file set yet
	CONFFILE=""

	# No commands specified yet
	COMMANDS=""

	# Force fetch to proceed
	FORCEFETCH=0

	# Run without a TTY
	NOTTYOK=0

	# Fetched first in a chain of commands
	ISFETCHED=0
}

# Parse the command line
parse_cmdline () {
	while [ $# -gt 0 ]; do
		case "$1" in
		# Location of configuration file
		-f)
			if [ $# -eq 1 ]; then usage; fi
			if [ ! -z "${CONFFILE}" ]; then usage; fi
			shift; CONFFILE="$1"
			;;
		-F)
			FORCEFETCH=1
			;;
		--not-running-from-cron)
			NOTTYOK=1
			;;
		--currently-running)
			shift
			config_SourceRelease $1 || usage
			;;

		# Configuration file equivalents
		-b)
			if [ $# -eq 1 ]; then usage; fi; shift
			config_BaseDir $1 || usage
			;;
		-d)
			if [ $# -eq 1 ]; then usage; fi; shift
			config_WorkDir $1 || usage
			;;
		-j)
			if [ $# -eq 1 ]; then usage; fi; shift
			config_TargetJail $1 || usage
			;;
		-k)
			if [ $# -eq 1 ]; then usage; fi; shift
			config_KeyPrint $1 || usage
			;;
		-s)
			if [ $# -eq 1 ]; then usage; fi; shift
			config_ServerName $1 || usage
			;;
		-r)
			if [ $# -eq 1 ]; then usage; fi; shift
			config_TargetRelease $1 || usage
			;;
		-t)
			if [ $# -eq 1 ]; then usage; fi; shift
			config_MailTo $1 || usage
			;;
		-v)
			if [ $# -eq 1 ]; then usage; fi; shift
			config_VerboseLevel $1 || usage
			;;

		# Aliases for "-v debug" and "-v nostats"
		--debug)
			config_VerboseLevel debug || usage
			;;
		--no-stats)
			config_VerboseLevel nostats || usage
			;;

		# Commands
		cron | fetch | upgrade | updatesready | install | rollback |\
		IDS | showconfig)
			COMMANDS="${COMMANDS} $1"
			;;

		# Anything else is an error
		*)
			usage
			;;
		esac
		shift
	done

	# Make sure we have at least one command
	if [ -z "${COMMANDS}" ]; then
		usage
	fi
}

# Parse the configuration file
parse_conffile () {
	# If a configuration file was specified on the command line, check
	# that it exists and is readable.
	if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then
		echo -n "File does not exist "
		echo -n "or is not readable: "
		echo ${CONFFILE}
		exit 1
	fi

	# If a configuration file was not specified on the command line,
	# use the default configuration file path.  If that default does
	# not exist, give up looking for any configuration.
	if [ -z "${CONFFILE}" ]; then
		CONFFILE="/etc/freebsd-update.conf"
		if [ ! -r "${CONFFILE}" ]; then
			return
		fi
	fi

	# Save the configuration options specified on the command line, and
	# clear all the options in preparation for reading the config file.
	saveconfig
	nullconfig

	# Read the configuration file.  Anything after the first '#' is
	# ignored, and any blank lines are ignored.
	L=0
	while read LINE; do
		L=$(($L + 1))
		LINEX=`echo "${LINE}" | cut -f 1 -d '#'`
		if ! configline ${LINEX}; then
			echo "Error processing configuration file, line $L:"
			echo "==> ${LINE}"
			exit 1
		fi
	done < ${CONFFILE}

	# Merge the settings read from the configuration file with those
	# provided at the command line.
	mergeconfig
}

# Provide some default parameters
default_params () {
	# Save any parameters already configured, and clear the slate
	saveconfig
	nullconfig

	# Default configurations
	config_WorkDir /var/db/freebsd-update
	config_MailTo root
	config_AllowAdd yes
	config_AllowDelete yes
	config_KeepModifiedMetadata yes
	config_BaseDir /
	config_VerboseLevel stats
	config_StrictComponents no
	config_BackupKernel yes
	config_BackupKernelDir /boot/kernel.old
	config_BackupKernelSymbolFiles no
	config_CreateBootEnv yes

	# Merge these defaults into the earlier-configured settings
	mergeconfig
}

# Set utility output filtering options, based on ${VERBOSELEVEL}
fetch_setup_verboselevel () {
	case ${VERBOSELEVEL} in
	debug)
		QUIETREDIR="/dev/stderr"
		QUIETFLAG=" "
		STATSREDIR="/dev/stderr"
		DDSTATS=".."
		XARGST="-t"
		NDEBUG=" "
		;;
	nostats)
		QUIETREDIR=""
		QUIETFLAG=""
		STATSREDIR="/dev/null"
		DDSTATS=".."
		XARGST=""
		NDEBUG=""
		;;
	stats)
		QUIETREDIR="/dev/null"
		QUIETFLAG="-q"
		STATSREDIR="/dev/stdout"
		DDSTATS=""
		XARGST=""
		NDEBUG="-n"
		;;
	esac
}

# Perform sanity checks and set some final parameters
# in preparation for fetching files.  Figure out which
# set of updates should be downloaded: If the user is
# running *-p[0-9]+, strip off the last part; if the
# user is running -SECURITY, call it -RELEASE.  Chdir
# into the working directory.
fetchupgrade_check_params () {
	export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"

	_SERVERNAME_z=\
"SERVERNAME must be given via command line or configuration file."
	_KEYPRINT_z="Key must be given via -k option or configuration file."
	_KEYPRINT_bad="Invalid key fingerprint: "
	_WORKDIR_bad="Directory does not exist or is not writable: "
	_WORKDIR_bad2="Directory is not on a persistent filesystem: "

	if [ -z "${SERVERNAME}" ]; then
		echo -n "`basename $0`: "
		echo "${_SERVERNAME_z}"
		exit 1
	fi
	if [ -z "${KEYPRINT}" ]; then
		echo -n "`basename $0`: "
		echo "${_KEYPRINT_z}"
		exit 1
	fi
	if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
		echo -n "`basename $0`: "
		echo -n "${_KEYPRINT_bad}"
		echo ${KEYPRINT}
		exit 1
	fi
	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
		echo -n "`basename $0`: "
		echo -n "${_WORKDIR_bad}"
		echo ${WORKDIR}
		exit 1
	fi
	case `df -T ${WORKDIR}` in */dev/md[0-9]* | *tmpfs*)
		echo -n "`basename $0`: "
		echo -n "${_WORKDIR_bad2}"
		echo ${WORKDIR}
		exit 1
		;;
	esac
	chmod 700 ${WORKDIR}
	cd ${WORKDIR} || exit 1

	# Generate release number.  The s/SECURITY/RELEASE/ bit exists
	# to provide an upgrade path for FreeBSD Update 1.x users, since
	# the kernels provided by FreeBSD Update 1.x are always labelled
	# as X.Y-SECURITY.
	RELNUM=`uname -r |
	    sed -E 's,-p[0-9]+,,' |
	    sed -E 's,-SECURITY,-RELEASE,'`
	ARCH=`uname -m`
	FETCHDIR=${RELNUM}/${ARCH}
	PATCHDIR=${RELNUM}/${ARCH}/bp

	# Disallow upgrade from a version that is not a release
	case ${RELNUM} in
	*-RELEASE | *-ALPHA*  | *-BETA* | *-RC*)
		;;
	*)
		echo -n "`basename $0`: "
		cat <<- EOF
			Cannot upgrade from a version that is not a release
			(including alpha, beta and release candidates)
			using `basename $0`. Instead, FreeBSD can be directly
			upgraded by source or upgraded to a RELEASE/RELENG version
			prior to running `basename $0`.
			Currently running: ${RELNUM}
		EOF
		exit 1
		;;
	esac

	# Figure out what directory contains the running kernel
	BOOTFILE=`sysctl -n kern.bootfile`
	KERNELDIR=${BOOTFILE%/kernel}
	if ! [ -d ${KERNELDIR} ]; then
		echo "Cannot identify running kernel"
		exit 1
	fi

	# Figure out what kernel configuration is running.  We start with
	# the output of `uname -i`, and then make the following adjustments:
	# 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
	# file says "ident SMP-GENERIC", I don't know...
	# 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
	# _and_ `sysctl kern.version` contains a line which ends "/SMP", then
	# we're running an SMP kernel.  This mis-identification is a bug
	# which was fixed in 6.2-STABLE.
	KERNCONF=`uname -i`
	if [ ${KERNCONF} = "SMP-GENERIC" ]; then
		KERNCONF=SMP
	fi
	if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
		if sysctl kern.version | grep -qE '/SMP$'; then
			KERNCONF=SMP
		fi
	fi

	# Define some paths
	BSPATCH=/usr/bin/bspatch
	SHA256=/sbin/sha256
	PHTTPGET=/usr/libexec/phttpget

	# Set up variables relating to VERBOSELEVEL
	fetch_setup_verboselevel

	# Construct a unique name from ${BASEDIR}
	BDHASH=`echo ${BASEDIR} | sha256 -q`
}

# Perform sanity checks etc. before fetching updates.
fetch_check_params () {
	fetchupgrade_check_params

	if ! [ -z "${TARGETRELEASE}" ]; then
		echo -n "`basename $0`: "
		echo -n "-r option is meaningless with 'fetch' command.  "
		echo "(Did you mean 'upgrade' instead?)"
		exit 1
	fi

	# Check that we have updates ready to install
	if [ -f ${BDHASH}-install/kerneldone -a $FORCEFETCH -eq 0 ]; then
		echo "You have a partially completed upgrade pending"
		echo "Run '$0 install' first."
		echo "Run '$0 fetch -F' to proceed anyway."
		exit 1
	fi
}

# Perform sanity checks etc. before fetching upgrades.
upgrade_check_params () {
	fetchupgrade_check_params

	# Unless set otherwise, we're upgrading to the same kernel config.
	NKERNCONF=${KERNCONF}

	# We need TARGETRELEASE set
	_TARGETRELEASE_z="Release target must be specified via -r option."
	if [ -z "${TARGETRELEASE}" ]; then
		echo -n "`basename $0`: "
		echo "${_TARGETRELEASE_z}"
		exit 1
	fi

	# The target release should be != the current release.
	if [ "${TARGETRELEASE}" = "${RELNUM}" ]; then
		echo -n "`basename $0`: "
		echo "Cannot upgrade from ${RELNUM} to itself"
		exit 1
	fi

	# Turning off AllowAdd or AllowDelete is a bad idea for upgrades.
	if [ "${ALLOWADD}" = "no" ]; then
		echo -n "`basename $0`: "
		echo -n "WARNING: \"AllowAdd no\" is a bad idea "
		echo "when upgrading between releases."
		echo
	fi
	if [ "${ALLOWDELETE}" = "no" ]; then
		echo -n "`basename $0`: "
		echo -n "WARNING: \"AllowDelete no\" is a bad idea "
		echo "when upgrading between releases."
		echo
	fi

	# Set EDITOR to /usr/bin/vi if it isn't already set
	: ${EDITOR:='/usr/bin/vi'}
}

# Perform sanity checks and set some final parameters in
# preparation for installing updates.
install_check_params () {
	# Check that we are root.  All sorts of things won't work otherwise.
	if [ `id -u` != 0 ]; then
		echo "You must be root to run this."
		exit 1
	fi

	# Check that securelevel <= 0.  Otherwise we can't update schg files.
	if [ `sysctl -n kern.securelevel` -gt 0 ]; then
		echo "Updates cannot be installed when the system securelevel"
		echo "is greater than zero."
		exit 1
	fi

	# Check that we have a working directory
	_WORKDIR_bad="Directory does not exist or is not writable: "
	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
		echo -n "`basename $0`: "
		echo -n "${_WORKDIR_bad}"
		echo ${WORKDIR}
		exit 1
	fi
	cd ${WORKDIR} || exit 1

	# Construct a unique name from ${BASEDIR}
	BDHASH=`echo ${BASEDIR} | sha256 -q`

	# Check that we have updates ready to install
	if ! [ -L ${BDHASH}-install ]; then
		echo "No updates are available to install."
		if [ $ISFETCHED -eq 0 ]; then
			echo "Run '$0 fetch' first."
			exit 2
		fi
		exit 0
	fi
	if ! [ -f ${BDHASH}-install/INDEX-OLD ] ||
	    ! [ -f ${BDHASH}-install/INDEX-NEW ]; then
		echo "Update manifest is corrupt -- this should never happen."
		echo "Re-run '$0 fetch'."
		exit 1
	fi

	# Figure out what directory contains the running kernel
	BOOTFILE=`sysctl -n kern.bootfile`
	KERNELDIR=${BOOTFILE%/kernel}
	if ! [ -d ${KERNELDIR} ]; then
		echo "Cannot identify running kernel"
		exit 1
	fi
}

# Creates a new boot environment
install_create_be () {
	# Figure out if we're running in a jail and return if we are
	if [ `sysctl -n security.jail.jailed` = 1 ]; then
		return 1
	fi
	# Operating on roots that aren't located at / will, more often than not,
	# not touch the boot environment.
	if [ "$BASEDIR" != "/" ]; then
		return 1
	fi
	# Create a boot environment if enabled
	if [ ${BOOTENV} = yes ]; then
		bectl check 2>/dev/null
		case $? in
			0)
				# Boot environment are supported
				CREATEBE=yes
				;;
			255)
				# Boot environments are not supported
				CREATEBE=no
				;;
			*)
				# If bectl returns an unexpected exit code, don't create a BE
				CREATEBE=no
				;;
		esac
		if [ ${CREATEBE} = yes ]; then
			echo -n "Creating snapshot of existing boot environment... "
			VERSION=`freebsd-version -ku | sort -V | tail -n 1`
			TIMESTAMP=`date +"%Y-%m-%d_%H%M%S"`
			bectl create -r ${VERSION}_${TIMESTAMP}
			if [ $? -eq 0 ]; then
				echo "done.";
			else
				echo "failed."
				exit 1
			fi
		fi
	fi
}

# Perform sanity checks and set some final parameters in
# preparation for UNinstalling updates.
rollback_check_params () {
	# Check that we are root.  All sorts of things won't work otherwise.
	if [ `id -u` != 0 ]; then
		echo "You must be root to run this."
		exit 1
	fi

	# Check that we have a working directory
	_WORKDIR_bad="Directory does not exist or is not writable: "
	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
		echo -n "`basename $0`: "
		echo -n "${_WORKDIR_bad}"
		echo ${WORKDIR}
		exit 1
	fi
	cd ${WORKDIR} || exit 1

	# Construct a unique name from ${BASEDIR}
	BDHASH=`echo ${BASEDIR} | sha256 -q`

	# Check that we have updates ready to rollback
	if ! [ -L ${BDHASH}-rollback ]; then
		echo "No rollback directory found."
		exit 1
	fi
	if ! [ -f ${BDHASH}-rollback/INDEX-OLD ] ||
	    ! [ -f ${BDHASH}-rollback/INDEX-NEW ]; then
		echo "Update manifest is corrupt -- this should never happen."
		exit 1
	fi
}

# Perform sanity checks and set some final parameters
# in preparation for comparing the system against the
# published index.  Figure out which index we should
# compare against: If the user is running *-p[0-9]+,
# strip off the last part; if the user is running
# -SECURITY, call it -RELEASE.  Chdir into the working
# directory.
IDS_check_params () {
	export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"

	_SERVERNAME_z=\
"SERVERNAME must be given via command line or configuration file."
	_KEYPRINT_z="Key must be given via -k option or configuration file."
	_KEYPRINT_bad="Invalid key fingerprint: "
	_WORKDIR_bad="Directory does not exist or is not writable: "

	if [ -z "${SERVERNAME}" ]; then
		echo -n "`basename $0`: "
		echo "${_SERVERNAME_z}"
		exit 1
	fi
	if [ -z "${KEYPRINT}" ]; then
		echo -n "`basename $0`: "
		echo "${_KEYPRINT_z}"
		exit 1
	fi
	if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
		echo -n "`basename $0`: "
		echo -n "${_KEYPRINT_bad}"
		echo ${KEYPRINT}
		exit 1
	fi
	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
		echo -n "`basename $0`: "
		echo -n "${_WORKDIR_bad}"
		echo ${WORKDIR}
		exit 1
	fi
	cd ${WORKDIR} || exit 1

	# Generate release number.  The s/SECURITY/RELEASE/ bit exists
	# to provide an upgrade path for FreeBSD Update 1.x users, since
	# the kernels provided by FreeBSD Update 1.x are always labelled
	# as X.Y-SECURITY.
	RELNUM=`uname -r |
	    sed -E 's,-p[0-9]+,,' |
	    sed -E 's,-SECURITY,-RELEASE,'`
	ARCH=`uname -m`
	FETCHDIR=${RELNUM}/${ARCH}
	PATCHDIR=${RELNUM}/${ARCH}/bp

	# Figure out what directory contains the running kernel
	BOOTFILE=`sysctl -n kern.bootfile`
	KERNELDIR=${BOOTFILE%/kernel}
	if ! [ -d ${KERNELDIR} ]; then
		echo "Cannot identify running kernel"
		exit 1
	fi

	# Figure out what kernel configuration is running.  We start with
	# the output of `uname -i`, and then make the following adjustments:
	# 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
	# file says "ident SMP-GENERIC", I don't know...
	# 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
	# _and_ `sysctl kern.version` contains a line which ends "/SMP", then
	# we're running an SMP kernel.  This mis-identification is a bug
	# which was fixed in 6.2-STABLE.
	KERNCONF=`uname -i`
	if [ ${KERNCONF} = "SMP-GENERIC" ]; then
		KERNCONF=SMP
	fi
	if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
		if sysctl kern.version | grep -qE '/SMP$'; then
			KERNCONF=SMP
		fi
	fi

	# Define some paths
	SHA256=/sbin/sha256
	PHTTPGET=/usr/libexec/phttpget

	# Set up variables relating to VERBOSELEVEL
	fetch_setup_verboselevel
}

#### Core functionality -- the actual work gets done here

# Use an SRV query to pick a server.  If the SRV query doesn't provide
# a useful answer, use the server name specified by the user.
# Put another way... look up _http._tcp.${SERVERNAME} and pick a server
# from that; or if no servers are returned, use ${SERVERNAME}.
# This allows a user to specify "portsnap.freebsd.org" (in which case
# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org"
# (in which case portsnap will use that particular server, since there
# won't be an SRV entry for that name).
#
# We ignore the Port field, since we are always going to use port 80.

# Fetch the mirror list, but do not pick a mirror yet.  Returns 1 if
# no mirrors are available for any reason.
fetch_pick_server_init () {
	: > serverlist_tried

# Check that host(1) exists (i.e., that the system wasn't built with the
# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist.
	if ! which -s host; then
		: > serverlist_full
		return 1
	fi

	echo -n "Looking up ${SERVERNAME} mirrors... "

# Issue the SRV query and pull out the Priority, Weight, and Target fields.
# BIND 9 prints "$name has SRV record ..." while BIND 8 prints
# "$name server selection ..."; we allow either format.
	MLIST="_http._tcp.${SERVERNAME}"
	host -t srv "${MLIST}" |
	    sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" |
	    cut -f 1,2,4 -d ' ' |
	    sed -e 's/\.$//' |
	    sort > serverlist_full

# If no records, give up -- we'll just use the server name we were given.
	if [ `wc -l < serverlist_full` -eq 0 ]; then
		echo "none found."
		return 1
	fi

# Report how many mirrors we found.
	echo `wc -l < serverlist_full` "mirrors found."

# Generate a random seed for use in picking mirrors.  If HTTP_PROXY
# is set, this will be used to generate the seed; otherwise, the seed
# will be random.
	if [ -n "${HTTP_PROXY}${http_proxy}" ]; then
		RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" |
		    tr -d 'a-f' |
		    cut -c 1-9`
	else
		RANDVALUE=`jot -r 1 0 999999999`
	fi
}

# Pick a mirror.  Returns 1 if we have run out of mirrors to try.
fetch_pick_server () {
# Generate a list of not-yet-tried mirrors
	sort serverlist_tried |
	    comm -23 serverlist_full - > serverlist

# Have we run out of mirrors?
	if [ `wc -l < serverlist` -eq 0 ]; then
		cat <<- EOF
			No mirrors remaining, giving up.

			This may be because upgrading from this platform (${ARCH})
			or release (${RELNUM}) is unsupported by `basename $0`. Only
			platforms with Tier 1 support can be upgraded by `basename $0`.
			See https://www.freebsd.org/platforms/ for more info.

			If unsupported, FreeBSD must be upgraded by source.
		EOF
		return 1
	fi

# Find the highest priority level (lowest numeric value).
	SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1`

# Add up the weights of the response lines at that priority level.
	SRV_WSUM=0;
	while read X; do
		case "$X" in
		${SRV_PRIORITY}\ *)
			SRV_W=`echo $X | cut -f 2 -d ' '`
			SRV_WSUM=$(($SRV_WSUM + $SRV_W))
			;;
		esac
	done < serverlist

# If all the weights are 0, pretend that they are all 1 instead.
	if [ ${SRV_WSUM} -eq 0 ]; then
		SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l`
		SRV_W_ADD=1
	else
		SRV_W_ADD=0
	fi

# Pick a value between 0 and the sum of the weights - 1
	SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}`

# Read through the list of mirrors and set SERVERNAME.  Write the line
# corresponding to the mirror we selected into serverlist_tried so that
# we won't try it again.
	while read X; do
		case "$X" in
		${SRV_PRIORITY}\ *)
			SRV_W=`echo $X | cut -f 2 -d ' '`
			SRV_W=$(($SRV_W + $SRV_W_ADD))
			if [ $SRV_RND -lt $SRV_W ]; then
				SERVERNAME=`echo $X | cut -f 3 -d ' '`
				echo "$X" >> serverlist_tried
				break
			else
				SRV_RND=$(($SRV_RND - $SRV_W))
			fi
			;;
		esac
	done < serverlist
}

# Take a list of ${oldhash}|${newhash} and output a list of needed patches,
# i.e., those for which we have ${oldhash} and don't have ${newhash}.
fetch_make_patchlist () {
	grep -vE "^([0-9a-f]{64})\|\1$" |
	    tr '|' ' ' |
		while read X Y; do
			if [ -f "files/${Y}.gz" ] ||
			    [ ! -f "files/${X}.gz" ]; then
				continue
			fi
			echo "${X}|${Y}"
		done | sort -u
}

# Print user-friendly progress statistics
fetch_progress () {
	LNC=0
	while read x; do
		LNC=$(($LNC + 1))
		if [ $(($LNC % 10)) = 0 ]; then
			echo -n $LNC
		elif [ $(($LNC % 2)) = 0 ]; then
			echo -n .
		fi
	done
	echo -n " "
}

# Function for asking the user if everything is ok
continuep () {
	while read -p "Does this look reasonable (y/n)? " CONTINUE; do
		case "${CONTINUE}" in
		[yY]*)
			return 0
			;;
		[nN]*)
			return 1
			;;
		esac
	done
}

# Initialize the working directory
workdir_init () {
	mkdir -p files
	touch tINDEX.present
}

# Check that we have a public key with an appropriate hash, or
# fetch the key if it doesn't exist.  Returns 1 if the key has
# not yet been fetched.
fetch_key () {
	if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
		return 0
	fi

	echo -n "Fetching public key from ${SERVERNAME}... "
	rm -f pub.ssl
	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/pub.ssl \
	    2>${QUIETREDIR} || true
	if ! [ -r pub.ssl ]; then
		echo "failed."
		return 1
	fi
	if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
		echo "key has incorrect hash."
		rm -f pub.ssl
		return 1
	fi
	echo "done."
}

# Fetch metadata signature, aka "tag".
fetch_tag () {
	echo -n "Fetching metadata signature "
	echo ${NDEBUG} "for ${RELNUM} from ${SERVERNAME}... "
	rm -f latest.ssl
	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/latest.ssl	\
	    2>${QUIETREDIR} || true
	if ! [ -r latest.ssl ]; then
		echo "failed."
		return 1
	fi

	openssl rsautl -pubin -inkey pub.ssl -verify		\
	    < latest.ssl > tag.new 2>${QUIETREDIR} || true
	rm latest.ssl

	if ! [ `wc -l < tag.new` = 1 ] ||
	    ! grep -qE	\
    "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
		tag.new; then
		echo "invalid signature."
		return 1
	fi

	echo "done."

	RELPATCHNUM=`cut -f 4 -d '|' < tag.new`
	TINDEXHASH=`cut -f 5 -d '|' < tag.new`
	EOLTIME=`cut -f 6 -d '|' < tag.new`
}

# Sanity-check the patch number in a tag, to make sure that we're not
# going to "update" backwards and to prevent replay attacks.
fetch_tagsanity () {
	# Check that we're not going to move from -pX to -pY with Y < X.
	RELPX=`uname -r | sed -E 's,.*-,,'`
	if echo ${RELPX} | grep -qE '^p[0-9]+$'; then
		RELPX=`echo ${RELPX} | cut -c 2-`
	else
		RELPX=0
	fi
	if [ "${RELPATCHNUM}" -lt "${RELPX}" ]; then
		echo
		echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
		echo " appear older than what"
		echo "we are currently running (`uname -r`)!"
		echo "Cowardly refusing to proceed any further."
		return 1
	fi

	# If "tag" exists and corresponds to ${RELNUM}, make sure that
	# it contains a patch number <= RELPATCHNUM, in order to protect
	# against rollback (replay) attacks.
	if [ -f tag ] &&
	    grep -qE	\
    "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
		tag; then
		LASTRELPATCHNUM=`cut -f 4 -d '|' < tag`

		if [ "${RELPATCHNUM}" -lt "${LASTRELPATCHNUM}" ]; then
			echo
			echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
			echo " are older than the"
			echo -n "most recently seen updates"
			echo " (${RELNUM}-p${LASTRELPATCHNUM})."
			echo "Cowardly refusing to proceed any further."
			return 1
		fi
	fi
}

# Fetch metadata index file
fetch_metadata_index () {
	echo ${NDEBUG} "Fetching metadata index... "
	rm -f ${TINDEXHASH}
	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/t/${TINDEXHASH}
	    2>${QUIETREDIR}
	if ! [ -f ${TINDEXHASH} ]; then
		echo "failed."
		return 1
	fi
	if [ `${SHA256} -q ${TINDEXHASH}` != ${TINDEXHASH} ]; then
		echo "update metadata index corrupt."
		return 1
	fi
	echo "done."
}

# Print an error message about signed metadata being bogus.
fetch_metadata_bogus () {
	echo
	echo "The update metadata$1 is correctly signed, but"
	echo "failed an integrity check."
	echo "Cowardly refusing to proceed any further."
	return 1
}

# Construct tINDEX.new by merging the lines named in $1 from ${TINDEXHASH}
# with the lines not named in $@ from tINDEX.present (if that file exists).
fetch_metadata_index_merge () {
	for METAFILE in $@; do
		if [ `grep -E "^${METAFILE}\|" ${TINDEXHASH} | wc -l`	\
		    -ne 1 ]; then
			fetch_metadata_bogus " index"
			return 1
		fi

		grep -E "${METAFILE}\|" ${TINDEXHASH}
	done |
	    sort > tINDEX.wanted

	if [ -f tINDEX.present ]; then
		join -t '|' -v 2 tINDEX.wanted tINDEX.present |
		    sort -m - tINDEX.wanted > tINDEX.new
		rm tINDEX.wanted
	else
		mv tINDEX.wanted tINDEX.new
	fi
}

# Sanity check all the lines of tINDEX.new.  Even if more metadata lines
# are added by future versions of the server, this won't cause problems,
# since the only lines which appear in tINDEX.new are the ones which we
# specifically grepped out of ${TINDEXHASH}.
fetch_metadata_index_sanity () {
	if grep -qvE '^[0-9A-Z.-]+\|[0-9a-f]{64}$' tINDEX.new; then
		fetch_metadata_bogus " index"
		return 1
	fi
}

# Sanity check the metadata file $1.
fetch_metadata_sanity () {
	# Some aliases to save space later: ${P} is a character which can
	# appear in a path; ${M} is the four numeric metadata fields; and
	# ${H} is a sha256 hash.
	P="[-+./:=,%@_[~[:alnum:]]"
	M="[0-9]+\|[0-9]+\|[0-9]+\|[0-9]+"
	H="[0-9a-f]{64}"

	# Check that the first four fields make sense.
	if gunzip -c < files/$1.gz |
	    grep -qvE "^[a-z]+\|[0-9a-z-]+\|${P}+\|[fdL-]\|"; then
		fetch_metadata_bogus ""
		return 1
	fi

	# Remove the first three fields.
	gunzip -c < files/$1.gz |
	    cut -f 4- -d '|' > sanitycheck.tmp

	# Sanity check entries with type 'f'
	if grep -E '^f' sanitycheck.tmp |
	    grep -qvE "^f\|${M}\|${H}\|${P}*\$"; then
		fetch_metadata_bogus ""
		return 1
	fi

	# Sanity check entries with type 'd'
	if grep -E '^d' sanitycheck.tmp |
	    grep -qvE "^d\|${M}\|\|\$"; then
		fetch_metadata_bogus ""
		return 1
	fi

	# Sanity check entries with type 'L'
	if grep -E '^L' sanitycheck.tmp |
	    grep -qvE "^L\|${M}\|${P}*\|\$"; then
		fetch_metadata_bogus ""
		return 1
	fi

	# Sanity check entries with type '-'
	if grep -E '^-' sanitycheck.tmp |
	    grep -qvE "^-\|\|\|\|\|\|"; then
		fetch_metadata_bogus ""
		return 1
	fi

	# Clean up
	rm sanitycheck.tmp
}

# Fetch the metadata index and metadata files listed in $@,
# taking advantage of metadata patches where possible.
fetch_metadata () {
	fetch_metadata_index || return 1
	fetch_metadata_index_merge $@ || return 1
	fetch_metadata_index_sanity || return 1

	# Generate a list of wanted metadata patches
	join -t '|' -o 1.2,2.2 tINDEX.present tINDEX.new |
	    fetch_make_patchlist > patchlist

	if [ -s patchlist ]; then
		# Attempt to fetch metadata patches
		echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
		echo ${NDEBUG} "metadata patches.${DDSTATS}"
		tr '|' '-' < patchlist |
		    lam -s "${FETCHDIR}/tp/" - -s ".gz" |
		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
			2>${STATSREDIR} | fetch_progress
		echo "done."

		# Attempt to apply metadata patches
		echo -n "Applying metadata patches... "
		tr '|' ' ' < patchlist |
		    while read X Y; do
			if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
			gunzip -c < ${X}-${Y}.gz > diff
			gunzip -c < files/${X}.gz > diff-OLD

			# Figure out which lines are being added and removed
			grep -E '^-' diff |
			    cut -c 2- |
			    while read PREFIX; do
				look "${PREFIX}" diff-OLD
			    done |
			    sort > diff-rm
			grep -E '^\+' diff |
			    cut -c 2- > diff-add

			# Generate the new file
			comm -23 diff-OLD diff-rm |
			    sort - diff-add > diff-NEW

			if [ `${SHA256} -q diff-NEW` = ${Y} ]; then
				mv diff-NEW files/${Y}
				gzip -n files/${Y}
			else
				mv diff-NEW ${Y}.bad
			fi
			rm -f ${X}-${Y}.gz diff
			rm -f diff-OLD diff-NEW diff-add diff-rm
		done 2>${QUIETREDIR}
		echo "done."
	fi

	# Update metadata without patches
	cut -f 2 -d '|' < tINDEX.new |
	    while read Y; do
		if [ ! -f "files/${Y}.gz" ]; then
			echo ${Y};
		fi
	    done |
	    sort -u > filelist

	if [ -s filelist ]; then
		echo -n "Fetching `wc -l < filelist | tr -d ' '` "
		echo ${NDEBUG} "metadata files... "
		lam -s "${FETCHDIR}/m/" - -s ".gz" < filelist |
		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
		    2>${QUIETREDIR}

		while read Y; do
			if ! [ -f ${Y}.gz ]; then
				echo "failed."
				return 1
			fi
			if [ `gunzip -c < ${Y}.gz |
			    ${SHA256} -q` = ${Y} ]; then
				mv ${Y}.gz files/${Y}.gz
			else
				echo "metadata is corrupt."
				return 1
			fi
		done < filelist
		echo "done."
	fi

# Sanity-check the metadata files.
	cut -f 2 -d '|' tINDEX.new > filelist
	while read X; do
		fetch_metadata_sanity ${X} || return 1
	done < filelist

# Remove files which are no longer needed
	cut -f 2 -d '|' tINDEX.present |
	    sort > oldfiles
	cut -f 2 -d '|' tINDEX.new |
	    sort |
	    comm -13 - oldfiles |
	    lam -s "files/" - -s ".gz" |
	    xargs rm -f
	rm patchlist filelist oldfiles
	rm ${TINDEXHASH}

# We're done!
	mv tINDEX.new tINDEX.present
	mv tag.new tag

	return 0
}

# Extract a subset of a downloaded metadata file containing only the parts
# which are listed in COMPONENTS.
fetch_filter_metadata_components () {
	METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
	gunzip -c < files/${METAHASH}.gz > $1.all

	# Fish out the lines belonging to components we care about.
	for C in ${COMPONENTS}; do
		look "`echo ${C} | tr '/' '|'`|" $1.all
	done > $1

	# Remove temporary file.
	rm $1.all
}

# Generate a filtered version of the metadata file $1 from the downloaded
# file, by fishing out the lines corresponding to components we're trying
# to keep updated, and then removing lines corresponding to paths we want
# to ignore.
fetch_filter_metadata () {
	# Fish out the lines belonging to components we care about.
	fetch_filter_metadata_components $1

	# Canonicalize directory names by removing any trailing / in
	# order to avoid listing directories multiple times if they
	# belong to multiple components.  Turning "/" into "" doesn't
	# matter, since we add a leading "/" when we use paths later.
	cut -f 3- -d '|' $1 |
	    sed -e 's,/|d|,|d|,' |
	    sed -e 's,/|-|,|-|,' |
	    sort -u > $1.tmp

	# Figure out which lines to ignore and remove them.
	for X in ${IGNOREPATHS}; do
		grep -E "^${X}" $1.tmp
	done |
	    sort -u |
	    comm -13 - $1.tmp > $1

	# Remove temporary files.
	rm $1.tmp
}

# Filter the metadata file $1 by adding lines with "/boot/$2"
# replaced by ${KERNELDIR} (which is `sysctl -n kern.bootfile` minus the
# trailing "/kernel"); and if "/boot/$2" does not exist, remove
# the original lines which start with that.
# Put another way: Deal with the fact that the FOO kernel is sometimes
# installed in /boot/FOO/ and is sometimes installed elsewhere.
fetch_filter_kernel_names () {
	grep ^/boot/$2 $1 |
	    sed -e "s,/boot/$2,${KERNELDIR},g" |
	    sort - $1 > $1.tmp
	mv $1.tmp $1

	if ! [ -d /boot/$2 ]; then
		grep -v ^/boot/$2 $1 > $1.tmp
		mv $1.tmp $1
	fi
}

# For all paths appearing in $1 or $3, inspect the system
# and generate $2 describing what is currently installed.
fetch_inspect_system () {
	# No errors yet...
	rm -f .err

	# Tell the user why his disk is suddenly making lots of noise
	echo -n "Inspecting system... "

	# Generate list of files to inspect
	cat $1 $3 |
	    cut -f 1 -d '|' |
	    sort -u > filelist

	# Examine each file and output lines of the form
	# /path/to/file|type|device-inum|user|group|perm|flags|value
	# sorted by device and inode number.
	while read F; do
		# If the symlink/file/directory does not exist, record this.
		if ! [ -e ${BASEDIR}/${F} ]; then
			echo "${F}|-||||||"
			continue
		fi
		if ! [ -r ${BASEDIR}/${F} ]; then
			echo "Cannot read file: ${BASEDIR}/${F}"	\
			    >/dev/stderr
			touch .err
			return 1
		fi

		# Otherwise, output an index line.
		if [ -L ${BASEDIR}/${F} ]; then
			echo -n "${F}|L|"
			stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
			readlink ${BASEDIR}/${F};
		elif [ -f ${BASEDIR}/${F} ]; then
			echo -n "${F}|f|"
			stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
			sha256 -q ${BASEDIR}/${F};
		elif [ -d ${BASEDIR}/${F} ]; then
			echo -n "${F}|d|"
			stat -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
		else
			echo "Unknown file type: ${BASEDIR}/${F}"	\
			    >/dev/stderr
			touch .err
			return 1
		fi
	done < filelist |
	    sort -k 3,3 -t '|' > $2.tmp
	rm filelist

	# Check if an error occurred during system inspection
	if [ -f .err ]; then
		return 1
	fi

	# Convert to the form
	# /path/to/file|type|user|group|perm|flags|value|hlink
	# by resolving identical device and inode numbers into hard links.
	cut -f 1,3 -d '|' $2.tmp |
	    sort -k 1,1 -t '|' |
	    sort -s -u -k 2,2 -t '|' |
	    join -1 2 -2 3 -t '|' - $2.tmp |
	    awk -F \| -v OFS=\|		\
		'{
		    if (($2 == $3) || ($4 == "-"))
			print $3,$4,$5,$6,$7,$8,$9,""
		    else
			print $3,$4,$5,$6,$7,$8,$9,$2
		}' |
	    sort > $2
	rm $2.tmp

	# We're finished looking around
	echo "done."
}

# For any paths matching ${MERGECHANGES}, compare $2 against $1 and $3 and
# find any files with values unique to $2; generate $4 containing these paths
# and their corresponding hashes from $1.
fetch_filter_mergechanges () {
	# Pull out the paths and hashes of the files matching ${MERGECHANGES}.
	for F in $1 $2 $3; do
		for X in ${MERGECHANGES}; do
			grep -E "^${X}" ${F}
		done |
		    cut -f 1,2,7 -d '|' |
		    sort > ${F}-values
	done

	# Any line in $2-values which doesn't appear in $1-values or $3-values
	# and is a file means that we should list the path in $3.
	sort $1-values $3-values |
	    comm -13 - $2-values |
	    fgrep '|f|' |
	    cut -f 1 -d '|' > $2-paths

	# For each path, pull out one (and only one!) entry from $1-values.
	# Note that we cannot distinguish which "old" version the user made
	# changes to; but hopefully any changes which occur due to security
	# updates will exist in both the "new" version and the version which
	# the user has installed, so the merging will still work.
	while read X; do
		look "${X}|" $1-values |
		    head -1
	done < $2-paths > $4

	# Clean up
	rm $1-values $2-values $3-values $2-paths
}

# For any paths matching ${UPDATEIFUNMODIFIED}, remove lines from $[123]
# which correspond to lines in $2 with hashes not matching $1 or $3, unless
# the paths are listed in $4.  For entries in $2 marked "not present"
# (aka. type -), remove lines from $[123] unless there is a corresponding
# entry in $1.
fetch_filter_unmodified_notpresent () {
	# Figure out which lines of $1 and $3 correspond to bits which
	# should only be updated if they haven't changed, and fish out
	# the (path, type, value) tuples.
	# NOTE: We don't consider a file to be "modified" if it matches
	# the hash from $3.
	for X in ${UPDATEIFUNMODIFIED}; do
		grep -E "^${X}" $1
		grep -E "^${X}" $3
	done |
	    cut -f 1,2,7 -d '|' |
	    sort > $1-values

	# Do the same for $2.
	for X in ${UPDATEIFUNMODIFIED}; do
		grep -E "^${X}" $2
	done |
	    cut -f 1,2,7 -d '|' |
	    sort > $2-values

	# Any entry in $2-values which is not in $1-values corresponds to
	# a path which we need to remove from $1, $2, and $3, unless it
	# that path appears in $4.
	comm -13 $1-values $2-values |
	    sort -t '|' -k 1,1 > mlines.tmp
	cut -f 1 -d '|' $4 |
	    sort |
	    join -v 2 -t '|' - mlines.tmp |
	    sort > mlines
	rm $1-values $2-values mlines.tmp

	# Any lines in $2 which are not in $1 AND are "not present" lines
	# also belong in mlines.
	comm -13 $1 $2 |
	    cut -f 1,2,7 -d '|' |
	    fgrep '|-|' >> mlines

	# Remove lines from $1, $2, and $3
	for X in $1 $2 $3; do
		sort -t '|' -k 1,1 ${X} > ${X}.tmp
		cut -f 1 -d '|' < mlines |
		    sort |
		    join -v 2 -t '|' - ${X}.tmp |
		    sort > ${X}
		rm ${X}.tmp
	done

	# Store a list of the modified files, for future reference
	fgrep -v '|-|' mlines |
	    cut -f 1 -d '|' > modifiedfiles
	rm mlines
}

# For each entry in $1 of type -, remove any corresponding
# entry from $2 if ${ALLOWADD} != "yes".  Remove all entries
# of type - from $1.
fetch_filter_allowadd () {
	cut -f 1,2 -d '|' < $1 |
	    fgrep '|-' |
	    cut -f 1 -d '|' > filesnotpresent

	if [ ${ALLOWADD} != "yes" ]; then
		sort < $2 |
		    join -v 1 -t '|' - filesnotpresent |
		    sort > $2.tmp
		mv $2.tmp $2
	fi

	sort < $1 |
	    join -v 1 -t '|' - filesnotpresent |
	    sort > $1.tmp
	mv $1.tmp $1
	rm filesnotpresent
}

# If ${ALLOWDELETE} != "yes", then remove any entries from $1
# which don't correspond to entries in $2.
fetch_filter_allowdelete () {
	# Produce a lists ${PATH}|${TYPE}
	for X in $1 $2; do
		cut -f 1-2 -d '|' < ${X} |
		    sort -u > ${X}.nodes
	done

	# Figure out which lines need to be removed from $1.
	if [ ${ALLOWDELETE} != "yes" ]; then
		comm -23 $1.nodes $2.nodes > $1.badnodes
	else
		: > $1.badnodes
	fi

	# Remove the relevant lines from $1
	while read X; do
		look "${X}|" $1
	done < $1.badnodes |
	    comm -13 - $1 > $1.tmp
	mv $1.tmp $1

	rm $1.badnodes $1.nodes $2.nodes
}

# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in $2
# with metadata not matching any entry in $1, replace the corresponding
# line of $3 with one having the same metadata as the entry in $2.
fetch_filter_modified_metadata () {
	# Fish out the metadata from $1 and $2
	for X in $1 $2; do
		cut -f 1-6 -d '|' < ${X} > ${X}.metadata
	done

	# Find the metadata we need to keep
	if [ ${KEEPMODIFIEDMETADATA} = "yes" ]; then
		comm -13 $1.metadata $2.metadata > keepmeta
	else
		: > keepmeta
	fi

	# Extract the lines which we need to remove from $3, and
	# construct the lines which we need to add to $3.
	: > $3.remove
	: > $3.add
	while read LINE; do
		NODE=`echo "${LINE}" | cut -f 1-2 -d '|'`
		look "${NODE}|" $3 >> $3.remove
		look "${NODE}|" $3 |
		    cut -f 7- -d '|' |
		    lam -s "${LINE}|" - >> $3.add
	done < keepmeta

	# Remove the specified lines and add the new lines.
	sort $3.remove |
	    comm -13 - $3 |
	    sort -u - $3.add > $3.tmp
	mv $3.tmp $3

	rm keepmeta $1.metadata $2.metadata $3.add $3.remove
}

# Remove lines from $1 and $2 which are identical;
# no need to update a file if it isn't changing.
fetch_filter_uptodate () {
	comm -23 $1 $2 > $1.tmp
	comm -13 $1 $2 > $2.tmp

	mv $1.tmp $1
	mv $2.tmp $2
}

# Fetch any "clean" old versions of files we need for merging changes.
fetch_files_premerge () {
	# We only need to do anything if $1 is non-empty.
	if [ -s $1 ]; then
		# Tell the user what we're doing
		echo -n "Fetching files from ${OLDRELNUM} for merging... "

		# List of files wanted
		fgrep '|f|' < $1 |
		    cut -f 3 -d '|' |
		    sort -u > files.wanted

		# Only fetch the files we don't already have
		while read Y; do
			if [ ! -f "files/${Y}.gz" ]; then
				echo ${Y};
			fi
		done < files.wanted > filelist

		# Actually fetch them
		lam -s "${OLDFETCHDIR}/f/" - -s ".gz" < filelist |
		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
		    2>${QUIETREDIR}

		# Make sure we got them all, and move them into /files/
		while read Y; do
			if ! [ -f ${Y}.gz ]; then
				echo "failed."
				return 1
			fi
			if [ `gunzip -c < ${Y}.gz |
			    ${SHA256} -q` = ${Y} ]; then
				mv ${Y}.gz files/${Y}.gz
			else
				echo "${Y} has incorrect hash."
				return 1
			fi
		done < filelist
		echo "done."

		# Clean up
		rm filelist files.wanted
	fi
}

# Prepare to fetch files: Generate a list of the files we need,
# copy the unmodified files we have into /files/, and generate
# a list of patches to download.
fetch_files_prepare () {
	# Tell the user why his disk is suddenly making lots of noise
	echo -n "Preparing to download files... "

	# Reduce indices to ${PATH}|${HASH} pairs
	for X in $1 $2 $3; do
		cut -f 1,2,7 -d '|' < ${X} |
		    fgrep '|f|' |
		    cut -f 1,3 -d '|' |
		    sort > ${X}.hashes
	done

	# List of files wanted
	cut -f 2 -d '|' < $3.hashes |
	    sort -u |
	    while read HASH; do
		if ! [ -f files/${HASH}.gz ]; then
			echo ${HASH}
		fi
	done > files.wanted

	# Generate a list of unmodified files
	comm -12 $1.hashes $2.hashes |
	    sort -k 1,1 -t '|' > unmodified.files

	# Copy all files into /files/.  We only need the unmodified files
	# for use in patching; but we'll want all of them if the user asks
	# to rollback the updates later.
	while read LINE; do
		F=`echo "${LINE}" | cut -f 1 -d '|'`
		HASH=`echo "${LINE}" | cut -f 2 -d '|'`

		# Skip files we already have.
		if [ -f files/${HASH}.gz ]; then
			continue
		fi

		# Make sure the file hasn't changed.
		cp "${BASEDIR}/${F}" tmpfile
		if [ `sha256 -q tmpfile` != ${HASH} ]; then
			echo
			echo "File changed while FreeBSD Update running: ${F}"
			return 1
		fi

		# Place the file into storage.
		gzip -c < tmpfile > files/${HASH}.gz
		rm tmpfile
	done < $2.hashes

	# Produce a list of patches to download
	sort -k 1,1 -t '|' $3.hashes |
	    join -t '|' -o 2.2,1.2 - unmodified.files |
	    fetch_make_patchlist > patchlist

	# Garbage collect
	rm unmodified.files $1.hashes $2.hashes $3.hashes

	# We don't need the list of possible old files any more.
	rm $1

	# We're finished making noise
	echo "done."
}

# Fetch files.
fetch_files () {
	# Attempt to fetch patches
	if [ -s patchlist ]; then
		echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
		echo ${NDEBUG} "patches.${DDSTATS}"
		tr '|' '-' < patchlist |
		    lam -s "${PATCHDIR}/" - |
		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
			2>${STATSREDIR} | fetch_progress
		echo "done."

		# Attempt to apply patches
		echo -n "Applying patches... "
		tr '|' ' ' < patchlist |
		    while read X Y; do
			if [ ! -f "${X}-${Y}" ]; then continue; fi
			gunzip -c < files/${X}.gz > OLD

			bspatch OLD NEW ${X}-${Y}

			if [ `${SHA256} -q NEW` = ${Y} ]; then
				mv NEW files/${Y}
				gzip -n files/${Y}
			fi
			rm -f diff OLD NEW ${X}-${Y}
		done 2>${QUIETREDIR}
		echo "done."
	fi

	# Download files which couldn't be generate via patching
	while read Y; do
		if [ ! -f "files/${Y}.gz" ]; then
			echo ${Y};
		fi
	done < files.wanted > filelist

	if [ -s filelist ]; then
		echo -n "Fetching `wc -l < filelist | tr -d ' '` "
		echo ${NDEBUG} "files... "
		lam -s "${FETCHDIR}/f/" - -s ".gz" < filelist |
		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
			2>${STATSREDIR} | fetch_progress

		while read Y; do
			if ! [ -f ${Y}.gz ]; then
				echo "failed."
				return 1
			fi
			if [ `gunzip -c < ${Y}.gz |
			    ${SHA256} -q` = ${Y} ]; then
				mv ${Y}.gz files/${Y}.gz
			else
				echo "${Y} has incorrect hash."
				return 1
			fi
		done < filelist
		echo "done."
	fi

	# Clean up
	rm files.wanted filelist patchlist
}

# Create and populate install manifest directory; and report what updates
# are available.
fetch_create_manifest () {
	# If we have an existing install manifest, nuke it.
	if [ -L "${BDHASH}-install" ]; then
		rm -r ${BDHASH}-install/
		rm ${BDHASH}-install
	fi

	# Report to the user if any updates were avoided due to local changes
	if [ -s modifiedfiles ]; then
		cat - modifiedfiles <<- EOF | ${PAGER}
			The following files are affected by updates. No changes have
			been downloaded, however, because the files have been modified
			locally:
		EOF
	fi
	rm modifiedfiles

	# If no files will be updated, tell the user and exit
	if ! [ -s INDEX-PRESENT ] &&
	    ! [ -s INDEX-NEW ]; then
		rm INDEX-PRESENT INDEX-NEW
		echo
		echo -n "No updates needed to update system to "
		echo "${RELNUM}-p${RELPATCHNUM}."
		return
	fi

	# Divide files into (a) removed files, (b) added files, and
	# (c) updated files.
	cut -f 1 -d '|' < INDEX-PRESENT |
	    sort > INDEX-PRESENT.flist
	cut -f 1 -d '|' < INDEX-NEW |
	    sort > INDEX-NEW.flist
	comm -23 INDEX-PRESENT.flist INDEX-NEW.flist > files.removed
	comm -13 INDEX-PRESENT.flist INDEX-NEW.flist > files.added
	comm -12 INDEX-PRESENT.flist INDEX-NEW.flist > files.updated
	rm INDEX-PRESENT.flist INDEX-NEW.flist

	# Report removed files, if any
	if [ -s files.removed ]; then
		cat - files.removed <<- EOF | ${PAGER}
			The following files will be removed as part of updating to
			${RELNUM}-p${RELPATCHNUM}:
		EOF
	fi
	rm files.removed

	# Report added files, if any
	if [ -s files.added ]; then
		cat - files.added <<- EOF | ${PAGER}
			The following files will be added as part of updating to
			${RELNUM}-p${RELPATCHNUM}:
		EOF
	fi
	rm files.added

	# Report updated files, if any
	if [ -s files.updated ]; then
		cat - files.updated <<- EOF | ${PAGER}
			The following files will be updated as part of updating to
			${RELNUM}-p${RELPATCHNUM}:
		EOF
	fi
	rm files.updated

	# Create a directory for the install manifest.
	MDIR=`mktemp -d install.XXXXXX` || return 1

	# Populate it
	mv INDEX-PRESENT ${MDIR}/INDEX-OLD
	mv INDEX-NEW ${MDIR}/INDEX-NEW

	# Link it into place
	ln -s ${MDIR} ${BDHASH}-install
}

# Warn about any upcoming EoL
fetch_warn_eol () {
	# What's the current time?
	NOWTIME=`date "+%s"`

	# When did we last warn about the EoL date?
	if [ -f lasteolwarn ]; then
		LASTWARN=`cat lasteolwarn`
	else
		LASTWARN=`expr ${NOWTIME} - 63072000`
	fi

	# If the EoL time is past, warn.
	if [ ${EOLTIME} -lt ${NOWTIME} ]; then
		echo
		cat <<-EOF
		WARNING: `uname -sr` HAS PASSED ITS END-OF-LIFE DATE.
		Any security issues discovered after `date -r ${EOLTIME}`
		will not have been corrected.
		EOF
		return 1
	fi

	# Figure out how long it has been since we last warned about the
	# upcoming EoL, and how much longer we have left.
	SINCEWARN=`expr ${NOWTIME} - ${LASTWARN}`
	TIMELEFT=`expr ${EOLTIME} - ${NOWTIME}`

	# Don't warn if the EoL is more than 3 months away
	if [ ${TIMELEFT} -gt 7884000 ]; then
		return 0
	fi

	# Don't warn if the time remaining is more than 3 times the time
	# since the last warning.
	if [ ${TIMELEFT} -gt `expr ${SINCEWARN} \* 3` ]; then
		return 0
	fi

	# Figure out what time units to use.
	if [ ${TIMELEFT} -lt 604800 ]; then
		UNIT="day"
		SIZE=86400
	elif [ ${TIMELEFT} -lt 2678400 ]; then
		UNIT="week"
		SIZE=604800
	else
		UNIT="month"
		SIZE=2678400
	fi

	# Compute the right number of units
	NUM=`expr ${TIMELEFT} / ${SIZE}`
	if [ ${NUM} != 1 ]; then
		UNIT="${UNIT}s"
	fi

	# Print the warning
	echo
	cat <<-EOF
		WARNING: `uname -sr` is approaching its End-of-Life date.
		It is strongly recommended that you upgrade to a newer
		release within the next ${NUM} ${UNIT}.
	EOF

	# Update the stored time of last warning
	echo ${NOWTIME} > lasteolwarn
}

# Do the actual work involved in "fetch" / "cron".
fetch_run () {
	workdir_init || return 1

	# Prepare the mirror list.
	fetch_pick_server_init && fetch_pick_server

	# Try to fetch the public key until we run out of servers.
	while ! fetch_key; do
		fetch_pick_server || return 1
	done

	# Try to fetch the metadata index signature ("tag") until we run
	# out of available servers; and sanity check the downloaded tag.
	while ! fetch_tag; do
		fetch_pick_server || return 1
	done
	fetch_tagsanity || return 1

	# Fetch the latest INDEX-NEW and INDEX-OLD files.
	fetch_metadata INDEX-NEW INDEX-OLD || return 1

	# Generate filtered INDEX-NEW and INDEX-OLD files containing only
	# the lines which (a) belong to components we care about, and (b)
	# don't correspond to paths we're explicitly ignoring.
	fetch_filter_metadata INDEX-NEW || return 1
	fetch_filter_metadata INDEX-OLD || return 1

	# Translate /boot/${KERNCONF} into ${KERNELDIR}
	fetch_filter_kernel_names INDEX-NEW ${KERNCONF}
	fetch_filter_kernel_names INDEX-OLD ${KERNCONF}

	# For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
	# system and generate an INDEX-PRESENT file.
	fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1

	# Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
	# correspond to lines in INDEX-PRESENT with hashes not appearing
	# in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
	# INDEX-PRESENT has type - and there isn't a corresponding entry in
	# INDEX-OLD with type -.
	fetch_filter_unmodified_notpresent	\
	    INDEX-OLD INDEX-PRESENT INDEX-NEW /dev/null

	# For each entry in INDEX-PRESENT of type -, remove any corresponding
	# entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
	# of type - from INDEX-PRESENT.
	fetch_filter_allowadd INDEX-PRESENT INDEX-NEW

	# If ${ALLOWDELETE} != "yes", then remove any entries from
	# INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
	fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW

	# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
	# INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
	# replace the corresponding line of INDEX-NEW with one having the
	# same metadata as the entry in INDEX-PRESENT.
	fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW

	# Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
	# no need to update a file if it isn't changing.
	fetch_filter_uptodate INDEX-PRESENT INDEX-NEW

	# Prepare to fetch files: Generate a list of the files we need,
	# copy the unmodified files we have into /files/, and generate
	# a list of patches to download.
	fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1

	# Fetch files.
	fetch_files || return 1

	# Create and populate install manifest directory; and report what
	# updates are available.
	fetch_create_manifest || return 1

	# Warn about any upcoming EoL
	fetch_warn_eol || return 1
}

# If StrictComponents is not "yes", generate a new components list
# with only the components which appear to be installed.
upgrade_guess_components () {
	if [ "${STRICTCOMPONENTS}" = "no" ]; then
		# Generate filtered INDEX-ALL with only the components listed
		# in COMPONENTS.
		fetch_filter_metadata_components $1 || return 1

		# Tell the user why his disk is suddenly making lots of noise
		echo -n "Inspecting system... "

		# Look at the files on disk, and assume that a component is
		# supposed to be present if it is more than half-present.
		cut -f 1-3 -d '|' < INDEX-ALL |
		    tr '|' ' ' |
		    while read C S F; do
			if [ -e ${BASEDIR}/${F} ]; then
				echo "+ ${C}|${S}"
			fi
			echo "= ${C}|${S}"
		    done |
		    sort |
		    uniq -c |
		    sed -E 's,^ +,,' > compfreq
		grep ' = ' compfreq |
		    cut -f 1,3 -d ' ' |
		    sort -k 2,2 -t ' ' > compfreq.total
		grep ' + ' compfreq |
		    cut -f 1,3 -d ' ' |
		    sort -k 2,2 -t ' ' > compfreq.present
		join -t ' ' -1 2 -2 2 compfreq.present compfreq.total |
		    while read S P T; do
			if [ ${T} -ne 0 -a ${P} -gt `expr ${T} / 2` ]; then
				echo ${S}
			fi
		    done > comp.present
		cut -f 2 -d ' ' < compfreq.total > comp.total
		rm INDEX-ALL compfreq compfreq.total compfreq.present

		# We're done making noise.
		echo "done."

		# Sometimes the kernel isn't installed where INDEX-ALL
		# thinks that it should be: In particular, it is often in
		# /boot/kernel instead of /boot/GENERIC or /boot/SMP.  To
		# deal with this, if "kernel|X" is listed in comp.total
		# (i.e., is a component which would be upgraded if it is
		# found to be present) we will add it to comp.present.
		# If "kernel|<anything>" is in comp.total but "kernel|X" is
		# not, we print a warning -- the user is running a kernel
		# which isn't part of the release.
		KCOMP=`echo ${KERNCONF} | tr 'A-Z' 'a-z'`
		grep -E "^kernel\|${KCOMP}\$" comp.total >> comp.present

		if grep -qE "^kernel\|" comp.total &&
		    ! grep -qE "^kernel\|${KCOMP}\$" comp.total; then
			cat <<-EOF

WARNING: This system is running a "${KCOMP}" kernel, which is not a
kernel configuration distributed as part of FreeBSD ${RELNUM}.
This kernel will not be updated: you MUST update the kernel manually
before running "$0 install".
			EOF
		fi

		# Re-sort the list of installed components and generate
		# the list of non-installed components.
		sort -u < comp.present > comp.present.tmp
		mv comp.present.tmp comp.present
		comm -13 comp.present comp.total > comp.absent

		# Ask the user to confirm that what we have is correct.  To
		# reduce user confusion, translate "X|Y" back to "X/Y" (as
		# subcomponents must be listed in the configuration file).
		echo
		echo -n "The following components of FreeBSD "
		echo "seem to be installed:"
		tr '|' '/' < comp.present |
		    fmt -72
		echo
		echo -n "The following components of FreeBSD "
		echo "do not seem to be installed:"
		tr '|' '/' < comp.absent |
		    fmt -72
		echo
		continuep || return 1
		echo

		# Suck the generated list of components into ${COMPONENTS}.
		# Note that comp.present.tmp is used due to issues with
		# pipelines and setting variables.
		COMPONENTS=""
		tr '|' '/' < comp.present > comp.present.tmp
		while read C; do
			COMPONENTS="${COMPONENTS} ${C}"
		done < comp.present.tmp

		# Delete temporary files
		rm comp.present comp.present.tmp comp.absent comp.total
	fi
}

# If StrictComponents is not "yes", COMPONENTS contains an entry
# corresponding to the currently running kernel, and said kernel
# does not exist in the new release, add "kernel/generic" to the
# list of components.
upgrade_guess_new_kernel () {
	if [ "${STRICTCOMPONENTS}" = "no" ]; then
		# Grab the unfiltered metadata file.
		METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
		gunzip -c < files/${METAHASH}.gz > $1.all

		# If "kernel/${KCOMP}" is in ${COMPONENTS} and that component
		# isn't in $1.all, we need to add kernel/generic.
		for C in ${COMPONENTS}; do
			if [ ${C} = "kernel/${KCOMP}" ] &&
			    ! grep -qE "^kernel\|${KCOMP}\|" $1.all; then
				COMPONENTS="${COMPONENTS} kernel/generic"
				NKERNCONF="GENERIC"
				cat <<-EOF

WARNING: This system is running a "${KCOMP}" kernel, which is not a
kernel configuration distributed as part of FreeBSD ${RELNUM}.
As part of upgrading to FreeBSD ${RELNUM}, this kernel will be
replaced with a "generic" kernel.
				EOF
				continuep || return 1
			fi
		done

		# Don't need this any more...
		rm $1.all
	fi
}

# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
upgrade_oldall_to_oldnew () {
	# For each ${F}|... which appears in INDEX-ALL but does not appear
	# in INDEX-OLD, add ${F}|-|||||| to INDEX-OLD.
	cut -f 1 -d '|' < $1 |
	    sort -u > $1.paths
	cut -f 1 -d '|' < $2 |
	    sort -u |
	    comm -13 $1.paths - |
	    lam - -s "|-||||||" |
	    sort - $1 > $1.tmp
	mv $1.tmp $1

	# Remove lines from INDEX-OLD which also appear in INDEX-ALL
	comm -23 $1 $2 > $1.tmp
	mv $1.tmp $1

	# Remove lines from INDEX-ALL which have a file name not appearing
	# anywhere in INDEX-OLD (since these must be files which haven't
	# changed -- if they were new, there would be an entry of type "-").
	cut -f 1 -d '|' < $1 |
	    sort -u > $1.paths
	sort -k 1,1 -t '|' < $2 |
	    join -t '|' - $1.paths |
	    sort > $2.tmp
	rm $1.paths
	mv $2.tmp $2

	# Rename INDEX-ALL to INDEX-NEW.
	mv $2 $3
}

# Helper for upgrade_merge: Return zero true iff the two files differ only
# in the contents of their RCS tags.
samef () {
	X=`sed -E 's/\\$FreeBSD.*\\$/\$FreeBSD\$/' < $1 | ${SHA256}`
	Y=`sed -E 's/\\$FreeBSD.*\\$/\$FreeBSD\$/' < $2 | ${SHA256}`

	if [ $X = $Y ]; then
		return 0;
	else
		return 1;
	fi
}

# From the list of "old" files in $1, merge changes in $2 with those in $3,
# and update $3 to reflect the hashes of merged files.
upgrade_merge () {
	# We only need to do anything if $1 is non-empty.
	if [ -s $1 ]; then
		cut -f 1 -d '|' $1 |
		    sort > $1-paths

		# Create staging area for merging files
		rm -rf merge/
		while read F; do
			D=`dirname ${F}`
			mkdir -p merge/old/${D}
			mkdir -p merge/${OLDRELNUM}/${D}
			mkdir -p merge/${RELNUM}/${D}
			mkdir -p merge/new/${D}
		done < $1-paths

		# Copy in files
		while read F; do
			# Currently installed file
			V=`look "${F}|" $2 | cut -f 7 -d '|'`
			gunzip < files/${V}.gz > merge/old/${F}

			# Old release
			if look "${F}|" $1 | fgrep -q "|f|"; then
				V=`look "${F}|" $1 | cut -f 3 -d '|'`
				gunzip < files/${V}.gz		\
				    > merge/${OLDRELNUM}/${F}
			fi

			# New release
			if look "${F}|" $3 | cut -f 1,2,7 -d '|' |
			    fgrep -q "|f|"; then
				V=`look "${F}|" $3 | cut -f 7 -d '|'`
				gunzip < files/${V}.gz		\
				    > merge/${RELNUM}/${F}
			fi
		done < $1-paths

		# Attempt to automatically merge changes
		echo -n "Attempting to automatically merge "
		echo -n "changes in files..."
		: > failed.merges
		while read F; do
			# If the file doesn't exist in the new release,
			# the result of "merging changes" is having the file
			# not exist.
			if ! [ -f merge/${RELNUM}/${F} ]; then
				continue
			fi

			# If the file didn't exist in the old release, we're
			# going to throw away the existing file and hope that
			# the version from the new release is what we want.
			if ! [ -f merge/${OLDRELNUM}/${F} ]; then
				cp merge/${RELNUM}/${F} merge/new/${F}
				continue
			fi

			# Some files need special treatment.
			case ${F} in
			/etc/spwd.db | /etc/pwd.db | /etc/login.conf.db)
				# Don't merge these -- we're rebuild them
				# after updates are installed.
				cp merge/old/${F} merge/new/${F}
				;;
			*)
				if ! diff3 -E -m -L "current version"	\
				    -L "${OLDRELNUM}" -L "${RELNUM}"	\
				    merge/old/${F}			\
				    merge/${OLDRELNUM}/${F}		\
				    merge/${RELNUM}/${F}		\
				    > merge/new/${F} 2>/dev/null; then
					echo ${F} >> failed.merges
				fi
				;;
			esac
		done < $1-paths
		echo " done."

		# Ask the user to handle any files which didn't merge.
		while read F; do
			# If the installed file differs from the version in
			# the old release only due to RCS tag expansion
			# then just use the version in the new release.
			if samef merge/old/${F} merge/${OLDRELNUM}/${F}; then
				cp merge/${RELNUM}/${F} merge/new/${F}
				continue
			fi

			cat <<-EOF

The following file could not be merged automatically: ${F}
Press Enter to edit this file in ${EDITOR} and resolve the conflicts
manually...
			EOF
			while true; do
				read response </dev/tty
				if expr "${response}" : '[Aa][Cc][Cc][Ee][Pp][Tt]' > /dev/null; then
					echo
					break
				fi
				${EDITOR} `pwd`/merge/new/${F} < /dev/tty

				if ! grep -qE '^(<<<<<<<|=======|>>>>>>>)([[:space:]].*)?$' $(pwd)/merge/new/${F} ; then
					break
				fi
				cat <<-EOF

Merge conflict markers remain in: ${F}
These must be resolved for the system to be functional.

Press Enter to return to editing this file, or type "ACCEPT" to carry on with
these lines remaining in the file.
				EOF
			done
		done < failed.merges
		rm failed.merges

		# Ask the user to confirm that he likes how the result
		# of merging files.
		while read F; do
			# Skip files which haven't changed except possibly
			# in their RCS tags.
			if [ -f merge/old/${F} ] && [ -f merge/new/${F} ] &&
			    samef merge/old/${F} merge/new/${F}; then
				continue
			fi

			# Skip files where the installed file differs from
			# the old file only due to RCS tags.
			if [ -f merge/old/${F} ] &&
			    [ -f merge/${OLDRELNUM}/${F} ] &&
			    samef merge/old/${F} merge/${OLDRELNUM}/${F}; then
				continue
			fi

			# Warn about files which are ceasing to exist.
			if ! [ -f merge/new/${F} ]; then
				cat <<-EOF

The following file will be removed, as it no longer exists in
FreeBSD ${RELNUM}: ${F}
				EOF
				continuep < /dev/tty || return 1
				continue
			fi

			# Print changes for the user's approval.
			cat <<-EOF

The following changes, which occurred between FreeBSD ${OLDRELNUM} and
FreeBSD ${RELNUM} have been merged into ${F}:
EOF
			diff -U 5 -L "current version" -L "new version"	\
			    merge/old/${F} merge/new/${F} || true
			continuep < /dev/tty || return 1
		done < $1-paths

		# Store merged files.
		while read F; do
			if [ -f merge/new/${F} ]; then
				V=`${SHA256} -q merge/new/${F}`

				gzip -c < merge/new/${F} > files/${V}.gz
				echo "${F}|${V}"
			fi
		done < $1-paths > newhashes

		# Pull lines out from $3 which need to be updated to
		# reflect merged files.
		while read F; do
			look "${F}|" $3
		done < $1-paths > $3-oldlines

		# Update lines to reflect merged files
		join -t '|' -o 1.1,1.2,1.3,1.4,1.5,1.6,2.2,1.8		\
		    $3-oldlines newhashes > $3-newlines

		# Remove old lines from $3 and add new lines.
		sort $3-oldlines |
		    comm -13 - $3 |
		    sort - $3-newlines > $3.tmp
		mv $3.tmp $3

		# Clean up
		rm $1-paths newhashes $3-oldlines $3-newlines
		rm -rf merge/
	fi

	# We're done with merging files.
	rm $1
}

# Do the work involved in fetching upgrades to a new release
upgrade_run () {
	workdir_init || return 1

	# Prepare the mirror list.
	fetch_pick_server_init && fetch_pick_server

	# Try to fetch the public key until we run out of servers.
	while ! fetch_key; do
		fetch_pick_server || return 1
	done

	# Try to fetch the metadata index signature ("tag") until we run
	# out of available servers; and sanity check the downloaded tag.
	while ! fetch_tag; do
		fetch_pick_server || return 1
	done
	fetch_tagsanity || return 1

	# Fetch the INDEX-OLD and INDEX-ALL.
	fetch_metadata INDEX-OLD INDEX-ALL || return 1

	# If StrictComponents is not "yes", generate a new components list
	# with only the components which appear to be installed.
	upgrade_guess_components INDEX-ALL || return 1

	# Generate filtered INDEX-OLD and INDEX-ALL files containing only
	# the components we want and without anything marked as "Ignore".
	fetch_filter_metadata INDEX-OLD || return 1
	fetch_filter_metadata INDEX-ALL || return 1

	# Merge the INDEX-OLD and INDEX-ALL files into INDEX-OLD.
	sort INDEX-OLD INDEX-ALL > INDEX-OLD.tmp
	mv INDEX-OLD.tmp INDEX-OLD
	rm INDEX-ALL

	# Adjust variables for fetching files from the new release.
	OLDRELNUM=${RELNUM}
	RELNUM=${TARGETRELEASE}
	OLDFETCHDIR=${FETCHDIR}
	FETCHDIR=${RELNUM}/${ARCH}

	# Try to fetch the NEW metadata index signature ("tag") until we run
	# out of available servers; and sanity check the downloaded tag.
	while ! fetch_tag; do
		fetch_pick_server || return 1
	done

	# Fetch the new INDEX-ALL.
	fetch_metadata INDEX-ALL || return 1

	# If StrictComponents is not "yes", COMPONENTS contains an entry
	# corresponding to the currently running kernel, and said kernel
	# does not exist in the new release, add "kernel/generic" to the
	# list of components.
	upgrade_guess_new_kernel INDEX-ALL || return 1

	# Filter INDEX-ALL to contain only the components we want and without
	# anything marked as "Ignore".
	fetch_filter_metadata INDEX-ALL || return 1

	# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
	# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
	upgrade_oldall_to_oldnew INDEX-OLD INDEX-ALL INDEX-NEW

	# Translate /boot/${KERNCONF} or /boot/${NKERNCONF} into ${KERNELDIR}
	fetch_filter_kernel_names INDEX-NEW ${NKERNCONF}
	fetch_filter_kernel_names INDEX-OLD ${KERNCONF}

	# For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
	# system and generate an INDEX-PRESENT file.
	fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1

	# Based on ${MERGECHANGES}, generate a file tomerge-old with the
	# paths and hashes of old versions of files to merge.
	fetch_filter_mergechanges INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old

	# Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
	# correspond to lines in INDEX-PRESENT with hashes not appearing
	# in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
	# INDEX-PRESENT has type - and there isn't a corresponding entry in
	# INDEX-OLD with type -.
	fetch_filter_unmodified_notpresent	\
	    INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old

	# For each entry in INDEX-PRESENT of type -, remove any corresponding
	# entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
	# of type - from INDEX-PRESENT.
	fetch_filter_allowadd INDEX-PRESENT INDEX-NEW

	# If ${ALLOWDELETE} != "yes", then remove any entries from
	# INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
	fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW

	# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
	# INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
	# replace the corresponding line of INDEX-NEW with one having the
	# same metadata as the entry in INDEX-PRESENT.
	fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW

	# Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
	# no need to update a file if it isn't changing.
	fetch_filter_uptodate INDEX-PRESENT INDEX-NEW

	# Fetch "clean" files from the old release for merging changes.
	fetch_files_premerge tomerge-old

	# Prepare to fetch files: Generate a list of the files we need,
	# copy the unmodified files we have into /files/, and generate
	# a list of patches to download.
	fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1

	# Fetch patches from to-${RELNUM}/${ARCH}/bp/
	PATCHDIR=to-${RELNUM}/${ARCH}/bp
	fetch_files || return 1

	# Merge configuration file changes.
	upgrade_merge tomerge-old INDEX-PRESENT INDEX-NEW || return 1

	# Create and populate install manifest directory; and report what
	# updates are available.
	fetch_create_manifest || return 1

	# Leave a note behind to tell the "install" command that the kernel
	# needs to be installed before the world.
	touch ${BDHASH}-install/kernelfirst

	# Remind the user that they need to run "freebsd-update install"
	# to install the downloaded bits, in case they didn't RTFM.
	echo "To install the downloaded upgrades, run \"$0 install\"."
}

# Make sure that all the file hashes mentioned in $@ have corresponding
# gzipped files stored in /files/.
install_verify () {
	# Generate a list of hashes
	cat $@ |
	    cut -f 2,7 -d '|' |
	    grep -E '^f' |
	    cut -f 2 -d '|' |
	    sort -u > filelist

	# Make sure all the hashes exist
	while read HASH; do
		if ! [ -f files/${HASH}.gz ]; then
			echo -n "Update files missing -- "
			echo "this should never happen."
			echo "Re-run '$0 fetch'."
			return 1
		fi
	done < filelist

	# Clean up
	rm filelist
}

# Remove the system immutable flag from files
install_unschg () {
	# Generate file list
	cat $@ |
	    cut -f 1 -d '|' > filelist

	# Remove flags
	while read F; do
		if ! [ -e ${BASEDIR}/${F} ]; then
			continue
		else
			echo ${BASEDIR}/${F}
		fi
	done < filelist | xargs chflags noschg || return 1

	# Clean up
	rm filelist
}

# Decide which directory name to use for kernel backups.
backup_kernel_finddir () {
	CNT=0
	while true ; do
		# Pathname does not exist, so it is OK use that name
		# for backup directory.
		if [ ! -e $BASEDIR/$BACKUPKERNELDIR ]; then
			return 0
		fi

		# If directory do exist, we only use if it has our
		# marker file.
		if [ -d $BASEDIR/$BACKUPKERNELDIR -a \
			-e $BASEDIR/$BACKUPKERNELDIR/.freebsd-update ]; then
			return 0
		fi

		# We could not use current directory name, so add counter to
		# the end and try again.
		CNT=$((CNT + 1))
		if [ $CNT -gt 9 ]; then
			echo "Could not find valid backup dir ($BASEDIR/$BACKUPKERNELDIR)"
			exit 1
		fi
		BACKUPKERNELDIR="`echo $BACKUPKERNELDIR | sed -Ee 's/[0-9]\$//'`"
		BACKUPKERNELDIR="${BACKUPKERNELDIR}${CNT}"
	done
}

# Backup the current kernel using hardlinks, if not disabled by user.
# Since we delete all files in the directory used for previous backups
# we create a marker file called ".freebsd-update" in the directory so
# we can determine on the next run that the directory was created by
# freebsd-update and we then do not accidentally remove user files in
# the unlikely case that the user has created a directory with a
# conflicting name.
backup_kernel () {
	# Only make kernel backup is so configured.
	if [ $BACKUPKERNEL != yes ]; then
		return 0
	fi

	# Decide which directory name to use for kernel backups.
	backup_kernel_finddir

	# Remove old kernel backup files.  If $BACKUPKERNELDIR was
	# "not ours", backup_kernel_finddir would have exited, so
	# deleting the directory content is as safe as we can make it.
	if [ -d $BASEDIR/$BACKUPKERNELDIR ]; then
		rm -fr $BASEDIR/$BACKUPKERNELDIR
	fi

	# Create directories for backup.
	mkdir -p $BASEDIR/$BACKUPKERNELDIR
	mtree -cdn -p "${BASEDIR}/${KERNELDIR}" | \
	    mtree -Ue -p "${BASEDIR}/${BACKUPKERNELDIR}" > /dev/null

	# Mark the directory as having been created by freebsd-update.
	touch $BASEDIR/$BACKUPKERNELDIR/.freebsd-update
	if [ $? -ne 0 ]; then
		echo "Could not create kernel backup directory"
		exit 1
	fi

	# Disable pathname expansion to be sure *.symbols is not
	# expanded.
	set -f

	# Use find to ignore symbol files, unless disabled by user.
	if [ $BACKUPKERNELSYMBOLFILES = yes ]; then
		FINDFILTER=""
	else
		FINDFILTER="-a ! -name *.debug -a ! -name *.symbols"
	fi

	# Backup all the kernel files using hardlinks.
	(cd ${BASEDIR}/${KERNELDIR} && find . -type f $FINDFILTER -exec \
	    cp -pl '{}' ${BASEDIR}/${BACKUPKERNELDIR}/'{}' \;)

	# Re-enable patchname expansion.
	set +f
}

# Check for and remove an existing directory that conflicts with the file or
# symlink that we are going to install.
dir_conflict () {
	if [ -d "$1" ]; then
		echo "Removing conflicting directory $1"
		rm -rf -- "$1"
	fi
}

# Install new files
install_from_index () {
	# First pass: Do everything apart from setting file flags.  We
	# can't set flags yet, because schg inhibits hard linking.
	sort -k 1,1 -t '|' $1 |
	    tr '|' ' ' |
	    while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
		case ${TYPE} in
		d)
			# Create a directory.  A file may change to a directory
			# on upgrade (PR273661).  If that happens, remove the
			# file first.
			if [ -e "${BASEDIR}/${FPATH}" ] && \
			    ! [ -d "${BASEDIR}/${FPATH}" ]; then
				rm -f -- "${BASEDIR}/${FPATH}"
			fi
			install -d -o ${OWNER} -g ${GROUP}		\
			    -m ${PERM} ${BASEDIR}/${FPATH}
			;;
		f)
			dir_conflict "${BASEDIR}/${FPATH}"
			if [ -z "${LINK}" ]; then
				# Create a file, without setting flags.
				gunzip < files/${HASH}.gz > ${HASH}
				install -S -o ${OWNER} -g ${GROUP}	\
				    -m ${PERM} ${HASH} ${BASEDIR}/${FPATH}
				rm ${HASH}
			else
				# Create a hard link.
				ln -f ${BASEDIR}/${LINK} ${BASEDIR}/${FPATH}
			fi
			;;
		L)
			dir_conflict "${BASEDIR}/${FPATH}"
			# Create a symlink
			ln -sfh ${HASH} ${BASEDIR}/${FPATH}
			;;
		esac
	    done

	# Perform a second pass, adding file flags.
	tr '|' ' ' < $1 |
	    while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
		if [ ${TYPE} = "f" ] &&
		    ! [ ${FLAGS} = "0" ]; then
			chflags ${FLAGS} ${BASEDIR}/${FPATH}
		fi
	    done
}

# Remove files which we want to delete
install_delete () {
	# Generate list of new files
	cut -f 1 -d '|' < $2 |
	    sort > newfiles

	# Generate subindex of old files we want to nuke
	sort -k 1,1 -t '|' $1 |
	    join -t '|' -v 1 - newfiles |
	    sort -r -k 1,1 -t '|' |
	    cut -f 1,2 -d '|' |
	    tr '|' ' ' > killfiles

	# Remove the offending bits
	while read FPATH TYPE; do
		case ${TYPE} in
		d)
			rmdir ${BASEDIR}/${FPATH}
			;;
		f)
			if [ -f "${BASEDIR}/${FPATH}" ]; then
				rm "${BASEDIR}/${FPATH}"
			fi
			;;
		L)
			if [ -L "${BASEDIR}/${FPATH}" ]; then
				rm "${BASEDIR}/${FPATH}"
			fi
			;;
		esac
	done < killfiles

	# Clean up
	rm newfiles killfiles
}

# Install new files, delete old files, and update generated files
install_files () {
	# If we haven't already dealt with the kernel, deal with it.
	if ! [ -f $1/kerneldone ]; then
		grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
		grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW

		# Backup current kernel before installing a new one
		backup_kernel || return 1

		# Install new files
		install_from_index INDEX-NEW || return 1

		# Remove files which need to be deleted
		install_delete INDEX-OLD INDEX-NEW || return 1

		# Update linker.hints if necessary
		if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
			kldxref -R ${BASEDIR}/boot/ 2>/dev/null
		fi

		# We've finished updating the kernel.
		touch $1/kerneldone

		# Do we need to ask for a reboot now?
		if [ -f $1/kernelfirst ] &&
		    [ -s INDEX-OLD -o -s INDEX-NEW ]; then
			cat <<-EOF

Kernel updates have been installed.  Please reboot and run
"$0 install" again to finish installing updates.
			EOF
			exit 0
		fi
	fi

	# If we haven't already dealt with the world, deal with it.
	if ! [ -f $1/worlddone ]; then
		# Create any necessary directories first
		grep -vE '^/boot/' $1/INDEX-NEW |
		    grep -E '^[^|]+\|d\|' > INDEX-NEW
		install_from_index INDEX-NEW || return 1

		# Install new runtime linker
		grep -vE '^/boot/' $1/INDEX-NEW |
		    grep -vE '^[^|]+\|d\|' |
		    grep -E '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' > INDEX-NEW
		install_from_index INDEX-NEW || return 1

		# Install new shared libraries next
		grep -vE '^/boot/' $1/INDEX-NEW |
		    grep -vE '^[^|]+\|d\|' |
		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
		install_from_index INDEX-NEW || return 1

		# Deal with everything else
		grep -vE '^/boot/' $1/INDEX-OLD |
		    grep -vE '^[^|]+\|d\|' |
		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
		    grep -vE '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-OLD
		grep -vE '^/boot/' $1/INDEX-NEW |
		    grep -vE '^[^|]+\|d\|' |
		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
		    grep -vE '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
		install_from_index INDEX-NEW || return 1
		install_delete INDEX-OLD INDEX-NEW || return 1

		# Restart host sshd if running (PR263489).  Note that this does
		# not affect child sshd processes handling existing sessions.
		if [ "$BASEDIR" = / ] && \
		    service sshd status >/dev/null 2>/dev/null; then
			echo
			echo "Restarting sshd after upgrade"
			service sshd restart
		fi

		# Rehash certs if we actually have certctl installed.
		if which certctl>/dev/null; then
			env DESTDIR=${BASEDIR} certctl rehash
		fi

		# Rebuild generated pwd files and /etc/login.conf.db.
		pwd_mkdb -d ${BASEDIR}/etc -p ${BASEDIR}/etc/master.passwd
		cap_mkdb ${BASEDIR}/etc/login.conf

		# Rebuild man page databases, if necessary.
		for D in /usr/share/man /usr/share/openssl/man; do
			if [ ! -d ${BASEDIR}/$D ]; then
				continue
			fi
			if [ -f ${BASEDIR}/$D/mandoc.db ] && \
			    [ -z "$(find ${BASEDIR}/$D -type f -newer ${BASEDIR}/$D/mandoc.db)" ]; then
				continue;
			fi
			makewhatis ${BASEDIR}/$D
		done

		# We've finished installing the world and deleting old files
		# which are not shared libraries.
		touch $1/worlddone

		# Do we need to ask the user to portupgrade now?
		grep -vE '^/boot/' $1/INDEX-NEW |
		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' |
		    cut -f 1 -d '|' |
		    sort > newfiles
		if grep -vE '^/boot/' $1/INDEX-OLD |
		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' |
		    cut -f 1 -d '|' |
		    sort |
		    join -v 1 - newfiles |
		    grep -q .; then
			cat <<-EOF

Completing this upgrade requires removing old shared object files.
Please rebuild all installed 3rd party software (e.g., programs
installed from the ports tree) and then run "$0 install"
again to finish installing updates.
			EOF
			rm newfiles
			exit 0
		fi
		rm newfiles
	fi

	# Remove old shared libraries
	grep -vE '^/boot/' $1/INDEX-NEW |
	    grep -vE '^[^|]+\|d\|' |
	    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
	grep -vE '^/boot/' $1/INDEX-OLD |
	    grep -vE '^[^|]+\|d\|' |
	    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-OLD
	install_delete INDEX-OLD INDEX-NEW || return 1

	# Remove old directories
	grep -vE '^/boot/' $1/INDEX-NEW |
	    grep -E '^[^|]+\|d\|' > INDEX-NEW
	grep -vE '^/boot/' $1/INDEX-OLD |
	    grep -E '^[^|]+\|d\|' > INDEX-OLD
	install_delete INDEX-OLD INDEX-NEW || return 1

	# Remove temporary files
	rm INDEX-OLD INDEX-NEW
}

# Rearrange bits to allow the installed updates to be rolled back
install_setup_rollback () {
	# Remove the "reboot after installing kernel", "kernel updated", and
	# "finished installing the world" flags if present -- they are
	# irrelevant when rolling back updates.
	if [ -f ${BDHASH}-install/kernelfirst ]; then
		rm ${BDHASH}-install/kernelfirst
		rm ${BDHASH}-install/kerneldone
	fi
	if [ -f ${BDHASH}-install/worlddone ]; then
		rm ${BDHASH}-install/worlddone
	fi

	if [ -L ${BDHASH}-rollback ]; then
		mv ${BDHASH}-rollback ${BDHASH}-install/rollback
	fi

	mv ${BDHASH}-install ${BDHASH}-rollback
}

# Actually install updates
install_run () {
	echo -n "Installing updates..."

	# Make sure we have all the files we should have
	install_verify ${BDHASH}-install/INDEX-OLD	\
	    ${BDHASH}-install/INDEX-NEW || return 1

	# Remove system immutable flag from files
	install_unschg ${BDHASH}-install/INDEX-OLD	\
	    ${BDHASH}-install/INDEX-NEW || return 1

	# Install new files, delete old files, and update linker.hints
	install_files ${BDHASH}-install || return 1

	# Rearrange bits to allow the installed updates to be rolled back
	install_setup_rollback

	echo " done."
}

# Rearrange bits to allow the previous set of updates to be rolled back next.
rollback_setup_rollback () {
	if [ -L ${BDHASH}-rollback/rollback ]; then
		mv ${BDHASH}-rollback/rollback rollback-tmp
		rm -r ${BDHASH}-rollback/
		rm ${BDHASH}-rollback
		mv rollback-tmp ${BDHASH}-rollback
	else
		rm -r ${BDHASH}-rollback/
		rm ${BDHASH}-rollback
	fi
}

# Install old files, delete new files, and update linker.hints
rollback_files () {
	# Create directories first.  They may be needed by files we will
	# install in subsequent steps (PR273950).
	awk -F \| '{if ($2 == "d") print }' $1/INDEX-OLD > INDEX-OLD
	install_from_index INDEX-OLD || return 1

	# Install old shared library files which don't have the same path as
	# a new shared library file.
	grep -vE '^/boot/' $1/INDEX-NEW |
	    grep -E '/lib/.*\.so\.[0-9]+\|' |
	    cut -f 1 -d '|' |
	    sort > INDEX-NEW.libs.flist
	grep -vE '^/boot/' $1/INDEX-OLD |
	    grep -E '/lib/.*\.so\.[0-9]+\|' |
	    sort -k 1,1 -t '|' - |
	    join -t '|' -v 1 - INDEX-NEW.libs.flist > INDEX-OLD
	install_from_index INDEX-OLD || return 1

	# Deal with files which are neither kernel nor shared library
	grep -vE '^/boot/' $1/INDEX-OLD |
	    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
	grep -vE '^/boot/' $1/INDEX-NEW |
	    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
	install_from_index INDEX-OLD || return 1
	install_delete INDEX-NEW INDEX-OLD || return 1

	# Install any old shared library files which we didn't install above.
	grep -vE '^/boot/' $1/INDEX-OLD |
	    grep -E '/lib/.*\.so\.[0-9]+\|' |
	    sort -k 1,1 -t '|' - |
	    join -t '|' - INDEX-NEW.libs.flist > INDEX-OLD
	install_from_index INDEX-OLD || return 1

	# Delete unneeded shared library files
	grep -vE '^/boot/' $1/INDEX-OLD |
	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
	grep -vE '^/boot/' $1/INDEX-NEW |
	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
	install_delete INDEX-NEW INDEX-OLD || return 1

	# Deal with kernel files
	grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
	grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
	install_from_index INDEX-OLD || return 1
	install_delete INDEX-NEW INDEX-OLD || return 1
	if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
		kldxref -R /boot/ 2>/dev/null
	fi

	# Remove temporary files
	rm INDEX-OLD INDEX-NEW INDEX-NEW.libs.flist
}

# Actually rollback updates
rollback_run () {
	echo -n "Uninstalling updates..."

	# If there are updates waiting to be installed, remove them; we
	# want the user to re-run 'fetch' after rolling back updates.
	if [ -L ${BDHASH}-install ]; then
		rm -r ${BDHASH}-install/
		rm ${BDHASH}-install
	fi

	# Make sure we have all the files we should have
	install_verify ${BDHASH}-rollback/INDEX-NEW	\
	    ${BDHASH}-rollback/INDEX-OLD || return 1

	# Remove system immutable flag from files
	install_unschg ${BDHASH}-rollback/INDEX-NEW	\
	    ${BDHASH}-rollback/INDEX-OLD || return 1

	# Install old files, delete new files, and update linker.hints
	rollback_files ${BDHASH}-rollback || return 1

	# Remove the rollback directory and the symlink pointing to it; and
	# rearrange bits to allow the previous set of updates to be rolled
	# back next.
	rollback_setup_rollback

	echo " done."
}

# Compare INDEX-ALL and INDEX-PRESENT and print warnings about differences.
IDS_compare () {
	# Get all the lines which mismatch in something other than file
	# flags.  We ignore file flags because sysinstall doesn't seem to
	# set them when it installs FreeBSD; warning about these adds a
	# very large amount of noise.
	cut -f 1-5,7-8 -d '|' $1 > $1.noflags
	sort -k 1,1 -t '|' $1.noflags > $1.sorted
	cut -f 1-5,7-8 -d '|' $2 |
	    comm -13 $1.noflags - |
	    fgrep -v '|-|||||' |
	    sort -k 1,1 -t '|' |
	    join -t '|' $1.sorted - > INDEX-NOTMATCHING

	# Ignore files which match IDSIGNOREPATHS.
	for X in ${IDSIGNOREPATHS}; do
		grep -E "^${X}" INDEX-NOTMATCHING
	done |
	    sort -u |
	    comm -13 - INDEX-NOTMATCHING > INDEX-NOTMATCHING.tmp
	mv INDEX-NOTMATCHING.tmp INDEX-NOTMATCHING

	# Go through the lines and print warnings.
	local IFS='|'
	while read FPATH TYPE OWNER GROUP PERM HASH LINK P_TYPE P_OWNER P_GROUP P_PERM P_HASH P_LINK; do
		# Warn about different object types.
		if ! [ "${TYPE}" = "${P_TYPE}" ]; then
			echo -n "${FPATH} is a "
			case "${P_TYPE}" in
			f)	echo -n "regular file, "
				;;
			d)	echo -n "directory, "
				;;
			L)	echo -n "symlink, "
				;;
			esac
			echo -n "but should be a "
			case "${TYPE}" in
			f)	echo -n "regular file."
				;;
			d)	echo -n "directory."
				;;
			L)	echo -n "symlink."
				;;
			esac
			echo

			# Skip other tests, since they don't make sense if
			# we're comparing different object types.
			continue
		fi

		# Warn about different owners.
		if ! [ "${OWNER}" = "${P_OWNER}" ]; then
			echo -n "${FPATH} is owned by user id ${P_OWNER}, "
			echo "but should be owned by user id ${OWNER}."
		fi

		# Warn about different groups.
		if ! [ "${GROUP}" = "${P_GROUP}" ]; then
			echo -n "${FPATH} is owned by group id ${P_GROUP}, "
			echo "but should be owned by group id ${GROUP}."
		fi

		# Warn about different permissions.  We do not warn about
		# different permissions on symlinks, since some archivers
		# don't extract symlink permissions correctly and they are
		# ignored anyway.
		if ! [ "${PERM}" = "${P_PERM}" ] &&
		    ! [ "${TYPE}" = "L" ]; then
			echo -n "${FPATH} has ${P_PERM} permissions, "
			echo "but should have ${PERM} permissions."
		fi

		# Warn about different file hashes / symlink destinations.
		if ! [ "${HASH}" = "${P_HASH}" ]; then
			if [ "${TYPE}" = "L" ]; then
				echo -n "${FPATH} is a symlink to ${P_HASH}, "
				echo "but should be a symlink to ${HASH}."
			fi
			if [ "${TYPE}" = "f" ]; then
				echo -n "${FPATH} has SHA256 hash ${P_HASH}, "
				echo "but should have SHA256 hash ${HASH}."
			fi
		fi

		# We don't warn about different hard links, since some
		# some archivers break hard links, and as long as the
		# underlying data is correct they really don't matter.
	done < INDEX-NOTMATCHING

	# Clean up
	rm $1 $1.noflags $1.sorted $2 INDEX-NOTMATCHING
}

# Do the work involved in comparing the system to a "known good" index
IDS_run () {
	workdir_init || return 1

	# Prepare the mirror list.
	fetch_pick_server_init && fetch_pick_server

	# Try to fetch the public key until we run out of servers.
	while ! fetch_key; do
		fetch_pick_server || return 1
	done

	# Try to fetch the metadata index signature ("tag") until we run
	# out of available servers; and sanity check the downloaded tag.
	while ! fetch_tag; do
		fetch_pick_server || return 1
	done
	fetch_tagsanity || return 1

	# Fetch INDEX-OLD and INDEX-ALL.
	fetch_metadata INDEX-OLD INDEX-ALL || return 1

	# Generate filtered INDEX-OLD and INDEX-ALL files containing only
	# the components we want and without anything marked as "Ignore".
	fetch_filter_metadata INDEX-OLD || return 1
	fetch_filter_metadata INDEX-ALL || return 1

	# Merge the INDEX-OLD and INDEX-ALL files into INDEX-ALL.
	sort INDEX-OLD INDEX-ALL > INDEX-ALL.tmp
	mv INDEX-ALL.tmp INDEX-ALL
	rm INDEX-OLD

	# Translate /boot/${KERNCONF} to ${KERNELDIR}
	fetch_filter_kernel_names INDEX-ALL ${KERNCONF}

	# Inspect the system and generate an INDEX-PRESENT file.
	fetch_inspect_system INDEX-ALL INDEX-PRESENT /dev/null || return 1

	# Compare INDEX-ALL and INDEX-PRESENT and print warnings about any
	# differences.
	IDS_compare INDEX-ALL INDEX-PRESENT
}

#### Main functions -- call parameter-handling and core functions

# Using the command line, configuration file, and defaults,
# set all the parameters which are needed later.
get_params () {
	init_params
	parse_cmdline $@
	parse_conffile
	default_params
}

# Fetch command.  Make sure that we're being called
# interactively, then run fetch_check_params and fetch_run
cmd_fetch () {
	finalize_components_config ${COMPONENTS}
	if [ ! -t 0 -a $NOTTYOK -eq 0 ]; then
		echo -n "`basename $0` fetch should not "
		echo "be run non-interactively."
		echo "Run `basename $0` cron instead."
		exit 1
	fi
	fetch_check_params
	fetch_run || exit 1
	ISFETCHED=1
}

# Cron command.  Make sure the parameters are sensible; wait
# rand(3600) seconds; then fetch updates.  While fetching updates,
# send output to a temporary file; only print that file if the
# fetching failed.
cmd_cron () {
	fetch_check_params
	sleep `jot -r 1 0 3600`

	TMPFILE=`mktemp /tmp/freebsd-update.XXXXXX` || exit 1
	finalize_components_config ${COMPONENTS} >> ${TMPFILE}
	if ! fetch_run >> ${TMPFILE} ||
	    ! grep -q "No updates needed" ${TMPFILE} ||
	    [ ${VERBOSELEVEL} = "debug" ]; then
		mail -s "`hostname` security updates" ${MAILTO} < ${TMPFILE}
	fi

	rm ${TMPFILE}
}

# Fetch files for upgrading to a new release.
cmd_upgrade () {
	finalize_components_config ${COMPONENTS}
	upgrade_check_params
	upgrade_run || exit 1
}

# Check if there are fetched updates ready to install.
# Chdir into the working directory.
cmd_updatesready () {
	finalize_components_config ${COMPONENTS}
	# Check if working directory exists (if not, no updates pending)
	if ! [ -e "${WORKDIR}" ]; then
		echo "No updates are available to install."
		exit 2
	fi

	# Change into working directory (fail if no permission/directory etc.)
	cd ${WORKDIR} || exit 1

	# Construct a unique name from ${BASEDIR}
	BDHASH=`echo ${BASEDIR} | sha256 -q`

	# Check that we have updates ready to install
	if ! [ -L ${BDHASH}-install ]; then
		echo "No updates are available to install."
		exit 2
	fi

	echo "There are updates available to install."
	echo "Run '$0 install' to proceed."
}

# Install downloaded updates.
cmd_install () {
	finalize_components_config ${COMPONENTS}
	install_check_params
	install_create_be
	install_run || exit 1
}

# Rollback most recently installed updates.
cmd_rollback () {
	finalize_components_config ${COMPONENTS}
	rollback_check_params
	rollback_run || exit 1
}

# Compare system against a "known good" index.
cmd_IDS () {
	finalize_components_config ${COMPONENTS}
	IDS_check_params
	IDS_run || exit 1
}

# Output configuration.
cmd_showconfig () {
	finalize_components_config ${COMPONENTS}
	for X in ${CONFIGOPTIONS}; do
		echo $X=$(eval echo \$${X})
	done
}

#### Entry point

# Make sure we find utilities from the base system
export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH}

# Set a pager if the user doesn't
if [ -z "$PAGER" ]; then
	PAGER=/usr/bin/less
fi

# Set LC_ALL in order to avoid problems with character ranges like [A-Z].
export LC_ALL=C

# Clear environment variables that may affect operation of tools that we use.
unset GREP_OPTIONS

get_params $@
for COMMAND in ${COMMANDS}; do
	cmd_${COMMAND}
done