xref: /freebsd/sys/contrib/openzfs/scripts/zimport.sh (revision ec0ea6efa1ad229d75c394c1a9b9cac33af2b1d3)
1#!/usr/bin/env bash
2#
3# Verify that an assortment of known good reference pools can be imported
4# using different versions of OpenZFS code.
5#
6# By default references pools for the major ZFS implementation will be
7# checked against the most recent OpenZFS tags and the master development branch.
8# Alternate tags or branches may be verified with the '-s <src-tag> option.
9# Passing the keyword "installed" will instruct the script to test whatever
10# version is installed.
11#
12# Preferentially a reference pool is used for all tests.  However, if one
13# does not exist and the pool-tag matches one of the src-tags then a new
14# reference pool will be created using binaries from that source build.
15# This is particularly useful when you need to test your changes before
16# opening a pull request.  The keyword 'all' can be used as short hand
17# refer to all available reference pools.
18#
19# New reference pools may be added by placing a bzip2 compressed tarball
20# of the pool in the scripts/zfs-images directory and then passing
21# the -p <pool-tag> option.  To increase the test coverage reference pools
22# should be collected for all the major ZFS implementations.  Having these
23# pools easily available is also helpful to the developers.
24#
25# Care should be taken to run these tests with a kernel supported by all
26# the listed tags.  Otherwise build failure will cause false positives.
27#
28#
29# EXAMPLES:
30#
31# The following example will verify the zfs-0.6.2 tag, the master branch,
32# and the installed zfs version can correctly import the listed pools.
33# Note there is no reference pool available for master and installed but
34# because binaries are available one is automatically constructed.  The
35# working directory is also preserved between runs (-k) preventing the
36# need to rebuild from source for multiple runs.
37#
38#  zimport.sh -k -f /var/tmp/zimport \
39#      -s "zfs-0.6.2 master installed" \
40#      -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed"
41#
42# ------------------------ OpenZFS Source Versions ----------------
43#                 zfs-0.6.2       master          0.6.2-175_g36eb554
44# -----------------------------------------------------------------
45# Clone ZFS       Local		Local		Skip
46# Build ZFS       Pass		Pass		Skip
47# -----------------------------------------------------------------
48# zevo-1.1.1      Pass		Pass		Pass
49# zol-0.6.2       Pass		Pass		Pass
50# zol-0.6.2-173   Fail		Pass		Pass
51# master          Pass		Pass		Pass
52# installed       Pass		Pass		Pass
53#
54
55BASE_DIR=$(dirname "$0")
56SCRIPT_COMMON=common.sh
57if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
58	. "${BASE_DIR}/${SCRIPT_COMMON}"
59else
60	echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
61fi
62
63PROG=zimport.sh
64SRC_TAGS="zfs-0.6.5.11 master"
65POOL_TAGS="all master"
66POOL_CREATE_OPTIONS=
67TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX)
68KEEP="no"
69VERBOSE="no"
70COLOR="yes"
71REPO="https://github.com/openzfs"
72IMAGES_DIR="$SCRIPTDIR/zfs-images/"
73IMAGES_TAR="https://github.com/openzfs/zfs-images/tarball/master"
74ERROR=0
75
76CONFIG_LOG="configure.log"
77CONFIG_OPTIONS=${CONFIG_OPTIONS:-""}
78MAKE_LOG="make.log"
79MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"}
80
81COLOR_GREEN="\033[0;32m"
82COLOR_RED="\033[0;31m"
83COLOR_BROWN="\033[0;33m"
84COLOR_RESET="\033[0m"
85
86usage() {
87cat << EOF
88USAGE:
89zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag]
90    [-f path] [-o options]
91
92DESCRIPTION:
93	ZPOOL import verification tests
94
95OPTIONS:
96	-h                Show this message
97	-v                Verbose
98	-c                No color
99	-k                Keep temporary directory
100	-r <repo>         Source repository ($REPO)
101	-s <src-tag>...   Verify OpenZFS versions with the listed tags
102	-i <pool-dir>     Pool image directory
103	-p <pool-tag>...  Verify pools created with the listed tags
104	-f <path>         Temporary directory to use
105	-o <options>      Additional options to pass to 'zpool create'
106
107EOF
108}
109
110while getopts 'hvckr:s:i:p:f:o:?' OPTION; do
111	case $OPTION in
112	h)
113		usage
114		exit 1
115		;;
116	v)
117		VERBOSE="yes"
118		;;
119	c)
120		COLOR="no"
121		;;
122	k)
123		KEEP="yes"
124		;;
125	r)
126		REPO="$OPTARG"
127		;;
128	s)
129		SRC_TAGS="$OPTARG"
130		;;
131	i)
132		IMAGES_DIR="$OPTARG"
133		;;
134	p)
135		POOL_TAGS="$OPTARG"
136		;;
137	f)
138		TEST_DIR="$OPTARG"
139		;;
140	o)
141		POOL_CREATE_OPTIONS="$OPTARG"
142		;;
143	?)
144		usage
145		exit 1
146		;;
147	esac
148done
149
150#
151# Verify the module start is not loaded
152#
153if lsmod | grep zfs >/dev/null; then
154	echo "ZFS modules must be unloaded"
155	exit 1
156fi
157
158#
159# Create a random directory tree of files and sub-directories to
160# to act as a copy source for the various regression tests.
161#
162populate() {
163	local ROOT=$1
164	local MAX_DIR_SIZE=$2
165	local MAX_FILE_SIZE=$3
166
167	mkdir -p "$ROOT"/{a,b,c,d,e,f,g}/{h,i}
168	DIRS=$(find "$ROOT")
169
170	for DIR in $DIRS; do
171		COUNT=$((RANDOM % MAX_DIR_SIZE))
172
173		for _ in $(seq $COUNT); do
174			FILE=$(mktemp -p "$DIR")
175			SIZE=$((RANDOM % MAX_FILE_SIZE))
176			dd if=/dev/urandom of="$FILE" bs=1k \
177			    count="$SIZE" &>/dev/null
178		done
179	done
180
181	return 0
182}
183
184SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX)
185trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT
186populate "$SRC_DIR" 10 100
187
188SRC_DIR="$TEST_DIR/src"
189SRC_DIR_ZFS="$SRC_DIR/zfs"
190
191if [ "$COLOR" = "no" ]; then
192	COLOR_GREEN=""
193	COLOR_BROWN=""
194	COLOR_RED=""
195	COLOR_RESET=""
196fi
197
198pass_nonewline() {
199	echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t"
200}
201
202skip_nonewline() {
203	echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t"
204}
205
206fail_nonewline() {
207	echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t"
208}
209
210#
211# Log a failure message, cleanup, and return an error.
212#
213fail() {
214	echo -e "$PROG: $1" >&2
215	$ZFS_SH -u >/dev/null 2>&1
216	exit 1
217}
218
219#
220# Set several helper variables which are derived from a source tag.
221#
222# ZFS_TAG - The passed zfs-x.y.z tag
223# ZFS_DIR - The zfs directory name
224# ZFS_URL - The zfs github URL to fetch the tarball
225#
226src_set_vars() {
227	local TAG=$1
228
229	ZFS_TAG="$TAG"
230	ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG"
231	ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG"
232
233	if [ "$TAG" = "installed" ]; then
234		ZPOOL_CMD=$(command -v zpool)
235		ZFS_CMD=$(command -v zfs)
236		ZFS_SH="/usr/share/zfs/zfs.sh"
237	else
238		ZPOOL_CMD="./cmd/zpool/zpool"
239		ZFS_CMD="./cmd/zfs/zfs"
240		ZFS_SH="./scripts/zfs.sh"
241	fi
242}
243
244#
245# Set several helper variables which are derived from a pool name such
246# as zol-0.6.x, zevo-1.1.1, etc.  These refer to example pools from various
247# ZFS implementations which are used to verify compatibility.
248#
249# POOL_TAG          - The example pools name in scripts/zfs-images/.
250# POOL_BZIP         - The full path to the example bzip2 compressed pool.
251# POOL_DIR          - The top level test path for this pool.
252# POOL_DIR_PRISTINE - The directory containing a pristine version of the pool.
253# POOL_DIR_COPY     - The directory containing a working copy of the pool.
254# POOL_DIR_SRC      - Location of a source build if it exists for this pool.
255#
256pool_set_vars() {
257	local TAG=$1
258
259	POOL_TAG=$TAG
260	POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2
261	POOL_DIR=$TEST_DIR/pools/$POOL_TAG
262	POOL_DIR_PRISTINE=$POOL_DIR/pristine
263	POOL_DIR_COPY=$POOL_DIR/copy
264	POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}"
265}
266
267#
268# Construct a non-trivial pool given a specific version of the source.  More
269# interesting pools provide better test coverage so this function should
270# extended as needed to create more realistic pools.
271#
272pool_create() {
273	pool_set_vars "$1"
274	src_set_vars "$1"
275
276	if [ "$POOL_TAG" != "installed" ]; then
277		cd "$POOL_DIR_SRC" || fail "Failed 'cd $POOL_DIR_SRC'"
278	fi
279
280	$ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || \
281	    fail "Failed to load kmods"
282
283	# Create a file vdev RAIDZ pool.
284	truncate -s 1G \
285	    "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
286	    "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \
287	    fail "Failed 'truncate -s 1G ...'"
288	# shellcheck disable=SC2086
289	$ZPOOL_CMD create $POOL_CREATE_OPTIONS "$POOL_TAG" raidz \
290	    "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
291	    "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \
292	    fail "Failed '$ZPOOL_CMD create $POOL_CREATE_OPTIONS $POOL_TAG ...'"
293
294	# Create a pool/fs filesystem with some random contents.
295	$ZFS_CMD create "$POOL_TAG/fs" || \
296	    fail "Failed '$ZFS_CMD create $POOL_TAG/fs'"
297	populate "/$POOL_TAG/fs/" 10 100
298
299	# Snapshot that filesystem, clone it, remove the files/dirs,
300	# replace them with new files/dirs.
301	$ZFS_CMD snap "$POOL_TAG/fs@snap" || \
302	    fail "Failed '$ZFS_CMD snap $POOL_TAG/fs@snap'"
303	$ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || \
304	    fail "Failed '$ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone'"
305	# shellcheck disable=SC2086
306	rm -Rf /$POOL_TAG/clone/*
307	populate "/$POOL_TAG/clone/" 10 100
308
309	# Scrub the pool, delay slightly, then export it.  It is now
310	# somewhat interesting for testing purposes.
311	$ZPOOL_CMD scrub "$POOL_TAG" || \
312	    fail "Failed '$ZPOOL_CMD scrub $POOL_TAG'"
313	sleep 10
314	$ZPOOL_CMD export "$POOL_TAG" || \
315	    fail "Failed '$ZPOOL_CMD export $POOL_TAG'"
316
317	$ZFS_SH -u || fail "Failed to unload kmods"
318}
319
320# If the zfs-images directory doesn't exist fetch a copy from Github then
321# cache it in the $TEST_DIR and update $IMAGES_DIR.
322if [ ! -d "$IMAGES_DIR" ]; then
323	IMAGES_DIR="$TEST_DIR/zfs-images"
324	mkdir -p "$IMAGES_DIR"
325	curl -sL "$IMAGES_TAR" | \
326	    tar -xz -C "$IMAGES_DIR" --strip-components=1 || \
327	    fail "Failed to download pool images"
328fi
329
330# Given the available images in the zfs-images directory substitute the
331# list of available images for the reserved keyword 'all'.
332for TAG in $POOL_TAGS; do
333
334	if  [ "$TAG" = "all" ]; then
335		ALL_TAGS=$(echo "$IMAGES_DIR"/*.tar.bz2 | \
336		    sed "s|$IMAGES_DIR/||g;s|.tar.bz2||g")
337		NEW_TAGS="$NEW_TAGS $ALL_TAGS"
338	else
339		NEW_TAGS="$NEW_TAGS $TAG"
340	fi
341done
342POOL_TAGS="$NEW_TAGS"
343
344if [ "$VERBOSE" = "yes" ]; then
345	echo "---------------------------- Options ----------------------------"
346	echo "VERBOSE=$VERBOSE"
347	echo "KEEP=$KEEP"
348	echo "REPO=$REPO"
349	echo "SRC_TAGS=$SRC_TAGS"
350	echo "POOL_TAGS=$POOL_TAGS"
351	echo "PATH=$TEST_DIR"
352	echo "POOL_CREATE_OPTIONS=$POOL_CREATE_OPTIONS"
353	echo
354fi
355
356if [ ! -d "$TEST_DIR" ]; then
357	mkdir -p "$TEST_DIR"
358fi
359
360if [ ! -d "$SRC_DIR" ]; then
361	mkdir -p "$SRC_DIR"
362fi
363
364# Print a header for all tags which are being tested.
365echo "------------------------ OpenZFS Source Versions ----------------"
366printf "%-16s" " "
367for TAG in $SRC_TAGS; do
368	src_set_vars "$TAG"
369
370	if [ "$TAG" = "installed" ]; then
371		ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }')
372		if [ -n "$ZFS_VERSION" ]; then
373			printf "%-16s" "$ZFS_VERSION"
374		else
375			fail "ZFS is not installed"
376		fi
377	else
378		printf "%-16s" "$TAG"
379	fi
380done
381echo -e "\n-----------------------------------------------------------------"
382
383#
384# Attempt to generate the tarball from your local git repository, if that
385# fails then attempt to download the tarball from Github.
386#
387printf "%-16s" "Clone ZFS"
388for TAG in $SRC_TAGS; do
389	src_set_vars "$TAG"
390
391	if [ -d "$ZFS_DIR" ]; then
392		skip_nonewline
393	elif  [ "$ZFS_TAG" = "installed" ]; then
394		skip_nonewline
395	else
396		cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'"
397
398		if [ ! -d "$SRC_DIR_ZFS" ]; then
399			mkdir -p "$SRC_DIR_ZFS"
400		fi
401
402		git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \
403		    -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/null || \
404		    rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
405		if [ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]; then
406			tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS"
407			rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
408			echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
409		else
410			mkdir -p "$ZFS_DIR" || fail "Failed to create $ZFS_DIR"
411			curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \
412			    --strip-components=1 || \
413			    fail "Failed to download $ZFS_URL"
414			echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
415		fi
416	fi
417done
418printf "\n"
419
420# Build the listed tags
421printf "%-16s" "Build ZFS"
422for TAG in $SRC_TAGS; do
423	src_set_vars "$TAG"
424
425	if [ -f "$ZFS_DIR/module/zfs/zfs.ko" ]; then
426		skip_nonewline
427	elif  [ "$ZFS_TAG" = "installed" ]; then
428		skip_nonewline
429	else
430		cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'"
431		make distclean &>/dev/null
432		./autogen.sh >>"$CONFIG_LOG" 2>&1 || \
433		    fail "Failed ZFS 'autogen.sh'"
434		# shellcheck disable=SC2086
435		./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || \
436		    fail "Failed ZFS 'configure $CONFIG_OPTIONS'"
437		# shellcheck disable=SC2086
438		make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \
439		    fail "Failed ZFS 'make $MAKE_OPTIONS'"
440		pass_nonewline
441	fi
442done
443printf "\n"
444echo "-----------------------------------------------------------------"
445
446# Either create a new pool using 'zpool create', or alternately restore an
447# existing pool from another ZFS implementation for compatibility testing.
448for TAG in $POOL_TAGS; do
449	pool_set_vars "$TAG"
450	SKIP=0
451
452	printf "%-16s" "$POOL_TAG"
453	rm -Rf "$POOL_DIR"
454	mkdir -p "$POOL_DIR_PRISTINE"
455
456	# Use the existing compressed image if available.
457	if [ -f "$POOL_BZIP" ]; then
458		tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \
459		    --strip-components=1 || \
460		    fail "Failed 'tar -xjf $POOL_BZIP"
461	# Use the installed version to create the pool.
462	elif  [ "$TAG" = "installed" ]; then
463		pool_create "$TAG"
464	# A source build is available to create the pool.
465	elif [ -d "$POOL_DIR_SRC" ]; then
466		pool_create "$TAG"
467	else
468		SKIP=1
469	fi
470
471	# Verify 'zpool import' works for all listed source versions.
472	for SRC_TAG in $SRC_TAGS; do
473
474		if [ $SKIP -eq 1 ]; then
475			skip_nonewline
476			continue
477		fi
478
479		src_set_vars "$SRC_TAG"
480		if [ "$SRC_TAG" != "installed" ]; then
481			cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'"
482		fi
483		$ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY"
484
485		cp -a --sparse=always "$POOL_DIR_PRISTINE" \
486		    "$POOL_DIR_COPY" || \
487		    fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY"
488		POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \
489		    awk '/pool:/ { print $2; exit }')
490
491		if ! $ZPOOL_CMD import -N -d "$POOL_DIR_COPY"
492		    "$POOL_NAME" &>/dev/null; then
493			fail_nonewline
494			ERROR=1
495		else
496			$ZPOOL_CMD export "$POOL_NAME" || \
497			    fail "Failed to export pool"
498			pass_nonewline
499		fi
500
501		rm -Rf "$POOL_DIR_COPY"
502
503		$ZFS_SH -u || fail "Failed to unload kmods"
504	done
505	printf "\n"
506done
507
508if [ "$KEEP" = "no" ]; then
509	rm -Rf "$TEST_DIR"
510fi
511
512exit $ERROR
513