xref: /freebsd/sys/contrib/openzfs/cmd/zpool/zpool_main.c (revision 65990b68a2cd89a08f0350e187df1968b16f4255)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
25  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
26  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
27  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
28  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
29  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
30  * Copyright (c) 2017 Datto Inc.
31  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
32  * Copyright (c) 2017, Intel Corporation.
33  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
34  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
35  * Copyright (c) 2021, Klara Inc.
36  * Copyright [2021] Hewlett Packard Enterprise Development LP
37  */
38 
39 #include <assert.h>
40 #include <ctype.h>
41 #include <dirent.h>
42 #include <errno.h>
43 #include <fcntl.h>
44 #include <getopt.h>
45 #include <libgen.h>
46 #include <libintl.h>
47 #include <libuutil.h>
48 #include <locale.h>
49 #include <pthread.h>
50 #include <stdio.h>
51 #include <stdlib.h>
52 #include <string.h>
53 #include <strings.h>
54 #include <time.h>
55 #include <unistd.h>
56 #include <pwd.h>
57 #include <zone.h>
58 #include <sys/wait.h>
59 #include <zfs_prop.h>
60 #include <sys/fs/zfs.h>
61 #include <sys/stat.h>
62 #include <sys/systeminfo.h>
63 #include <sys/fm/fs/zfs.h>
64 #include <sys/fm/util.h>
65 #include <sys/fm/protocol.h>
66 #include <sys/zfs_ioctl.h>
67 #include <sys/mount.h>
68 #include <sys/sysmacros.h>
69 
70 #include <math.h>
71 
72 #include <libzfs.h>
73 #include <libzutil.h>
74 
75 #include "zpool_util.h"
76 #include "zfs_comutil.h"
77 #include "zfeature_common.h"
78 
79 #include "statcommon.h"
80 
81 libzfs_handle_t *g_zfs;
82 
83 static int zpool_do_create(int, char **);
84 static int zpool_do_destroy(int, char **);
85 
86 static int zpool_do_add(int, char **);
87 static int zpool_do_remove(int, char **);
88 static int zpool_do_labelclear(int, char **);
89 
90 static int zpool_do_checkpoint(int, char **);
91 
92 static int zpool_do_list(int, char **);
93 static int zpool_do_iostat(int, char **);
94 static int zpool_do_status(int, char **);
95 
96 static int zpool_do_online(int, char **);
97 static int zpool_do_offline(int, char **);
98 static int zpool_do_clear(int, char **);
99 static int zpool_do_reopen(int, char **);
100 
101 static int zpool_do_reguid(int, char **);
102 
103 static int zpool_do_attach(int, char **);
104 static int zpool_do_detach(int, char **);
105 static int zpool_do_replace(int, char **);
106 static int zpool_do_split(int, char **);
107 
108 static int zpool_do_initialize(int, char **);
109 static int zpool_do_scrub(int, char **);
110 static int zpool_do_resilver(int, char **);
111 static int zpool_do_trim(int, char **);
112 
113 static int zpool_do_import(int, char **);
114 static int zpool_do_export(int, char **);
115 
116 static int zpool_do_upgrade(int, char **);
117 
118 static int zpool_do_history(int, char **);
119 static int zpool_do_events(int, char **);
120 
121 static int zpool_do_get(int, char **);
122 static int zpool_do_set(int, char **);
123 
124 static int zpool_do_sync(int, char **);
125 
126 static int zpool_do_version(int, char **);
127 
128 static int zpool_do_wait(int, char **);
129 
130 static zpool_compat_status_t zpool_do_load_compat(
131     const char *, boolean_t *);
132 
133 /*
134  * These libumem hooks provide a reasonable set of defaults for the allocator's
135  * debugging facilities.
136  */
137 
138 #ifdef DEBUG
139 const char *
140 _umem_debug_init(void)
141 {
142 	return ("default,verbose"); /* $UMEM_DEBUG setting */
143 }
144 
145 const char *
146 _umem_logging_init(void)
147 {
148 	return ("fail,contents"); /* $UMEM_LOGGING setting */
149 }
150 #endif
151 
152 typedef enum {
153 	HELP_ADD,
154 	HELP_ATTACH,
155 	HELP_CLEAR,
156 	HELP_CREATE,
157 	HELP_CHECKPOINT,
158 	HELP_DESTROY,
159 	HELP_DETACH,
160 	HELP_EXPORT,
161 	HELP_HISTORY,
162 	HELP_IMPORT,
163 	HELP_IOSTAT,
164 	HELP_LABELCLEAR,
165 	HELP_LIST,
166 	HELP_OFFLINE,
167 	HELP_ONLINE,
168 	HELP_REPLACE,
169 	HELP_REMOVE,
170 	HELP_INITIALIZE,
171 	HELP_SCRUB,
172 	HELP_RESILVER,
173 	HELP_TRIM,
174 	HELP_STATUS,
175 	HELP_UPGRADE,
176 	HELP_EVENTS,
177 	HELP_GET,
178 	HELP_SET,
179 	HELP_SPLIT,
180 	HELP_SYNC,
181 	HELP_REGUID,
182 	HELP_REOPEN,
183 	HELP_VERSION,
184 	HELP_WAIT
185 } zpool_help_t;
186 
187 
188 /*
189  * Flags for stats to display with "zpool iostats"
190  */
191 enum iostat_type {
192 	IOS_DEFAULT = 0,
193 	IOS_LATENCY = 1,
194 	IOS_QUEUES = 2,
195 	IOS_L_HISTO = 3,
196 	IOS_RQ_HISTO = 4,
197 	IOS_COUNT,	/* always last element */
198 };
199 
200 /* iostat_type entries as bitmasks */
201 #define	IOS_DEFAULT_M	(1ULL << IOS_DEFAULT)
202 #define	IOS_LATENCY_M	(1ULL << IOS_LATENCY)
203 #define	IOS_QUEUES_M	(1ULL << IOS_QUEUES)
204 #define	IOS_L_HISTO_M	(1ULL << IOS_L_HISTO)
205 #define	IOS_RQ_HISTO_M	(1ULL << IOS_RQ_HISTO)
206 
207 /* Mask of all the histo bits */
208 #define	IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M)
209 
210 /*
211  * Lookup table for iostat flags to nvlist names.  Basically a list
212  * of all the nvlists a flag requires.  Also specifies the order in
213  * which data gets printed in zpool iostat.
214  */
215 static const char *vsx_type_to_nvlist[IOS_COUNT][15] = {
216 	[IOS_L_HISTO] = {
217 	    ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
218 	    ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
219 	    ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
220 	    ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
221 	    ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
222 	    ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
223 	    ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
224 	    ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
225 	    ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
226 	    ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
227 	    ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
228 	    NULL},
229 	[IOS_LATENCY] = {
230 	    ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
231 	    ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
232 	    ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
233 	    ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
234 	    ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
235 	    ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
236 	    NULL},
237 	[IOS_QUEUES] = {
238 	    ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
239 	    ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
240 	    ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
241 	    ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
242 	    ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
243 	    ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
244 	    ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
245 	    NULL},
246 	[IOS_RQ_HISTO] = {
247 	    ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
248 	    ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
249 	    ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
250 	    ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
251 	    ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
252 	    ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
253 	    ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
254 	    ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
255 	    ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
256 	    ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
257 	    ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
258 	    ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
259 	    ZPOOL_CONFIG_VDEV_IND_REBUILD_HISTO,
260 	    ZPOOL_CONFIG_VDEV_AGG_REBUILD_HISTO,
261 	    NULL},
262 };
263 
264 
265 /*
266  * Given a cb->cb_flags with a histogram bit set, return the iostat_type.
267  * Right now, only one histo bit is ever set at one time, so we can
268  * just do a highbit64(a)
269  */
270 #define	IOS_HISTO_IDX(a)	(highbit64(a & IOS_ANYHISTO_M) - 1)
271 
272 typedef struct zpool_command {
273 	const char	*name;
274 	int		(*func)(int, char **);
275 	zpool_help_t	usage;
276 } zpool_command_t;
277 
278 /*
279  * Master command table.  Each ZFS command has a name, associated function, and
280  * usage message.  The usage messages need to be internationalized, so we have
281  * to have a function to return the usage message based on a command index.
282  *
283  * These commands are organized according to how they are displayed in the usage
284  * message.  An empty command (one with a NULL name) indicates an empty line in
285  * the generic usage message.
286  */
287 static zpool_command_t command_table[] = {
288 	{ "version",	zpool_do_version,	HELP_VERSION		},
289 	{ NULL },
290 	{ "create",	zpool_do_create,	HELP_CREATE		},
291 	{ "destroy",	zpool_do_destroy,	HELP_DESTROY		},
292 	{ NULL },
293 	{ "add",	zpool_do_add,		HELP_ADD		},
294 	{ "remove",	zpool_do_remove,	HELP_REMOVE		},
295 	{ NULL },
296 	{ "labelclear",	zpool_do_labelclear,	HELP_LABELCLEAR		},
297 	{ NULL },
298 	{ "checkpoint",	zpool_do_checkpoint,	HELP_CHECKPOINT		},
299 	{ NULL },
300 	{ "list",	zpool_do_list,		HELP_LIST		},
301 	{ "iostat",	zpool_do_iostat,	HELP_IOSTAT		},
302 	{ "status",	zpool_do_status,	HELP_STATUS		},
303 	{ NULL },
304 	{ "online",	zpool_do_online,	HELP_ONLINE		},
305 	{ "offline",	zpool_do_offline,	HELP_OFFLINE		},
306 	{ "clear",	zpool_do_clear,		HELP_CLEAR		},
307 	{ "reopen",	zpool_do_reopen,	HELP_REOPEN		},
308 	{ NULL },
309 	{ "attach",	zpool_do_attach,	HELP_ATTACH		},
310 	{ "detach",	zpool_do_detach,	HELP_DETACH		},
311 	{ "replace",	zpool_do_replace,	HELP_REPLACE		},
312 	{ "split",	zpool_do_split,		HELP_SPLIT		},
313 	{ NULL },
314 	{ "initialize",	zpool_do_initialize,	HELP_INITIALIZE		},
315 	{ "resilver",	zpool_do_resilver,	HELP_RESILVER		},
316 	{ "scrub",	zpool_do_scrub,		HELP_SCRUB		},
317 	{ "trim",	zpool_do_trim,		HELP_TRIM		},
318 	{ NULL },
319 	{ "import",	zpool_do_import,	HELP_IMPORT		},
320 	{ "export",	zpool_do_export,	HELP_EXPORT		},
321 	{ "upgrade",	zpool_do_upgrade,	HELP_UPGRADE		},
322 	{ "reguid",	zpool_do_reguid,	HELP_REGUID		},
323 	{ NULL },
324 	{ "history",	zpool_do_history,	HELP_HISTORY		},
325 	{ "events",	zpool_do_events,	HELP_EVENTS		},
326 	{ NULL },
327 	{ "get",	zpool_do_get,		HELP_GET		},
328 	{ "set",	zpool_do_set,		HELP_SET		},
329 	{ "sync",	zpool_do_sync,		HELP_SYNC		},
330 	{ NULL },
331 	{ "wait",	zpool_do_wait,		HELP_WAIT		},
332 };
333 
334 #define	NCOMMAND	(ARRAY_SIZE(command_table))
335 
336 #define	VDEV_ALLOC_CLASS_LOGS	"logs"
337 
338 static zpool_command_t *current_command;
339 static zfs_type_t current_prop_type = (ZFS_TYPE_POOL | ZFS_TYPE_VDEV);
340 static char history_str[HIS_MAX_RECORD_LEN];
341 static boolean_t log_history = B_TRUE;
342 static uint_t timestamp_fmt = NODATE;
343 
344 static const char *
345 get_usage(zpool_help_t idx)
346 {
347 	switch (idx) {
348 	case HELP_ADD:
349 		return (gettext("\tadd [-fgLnP] [-o property=value] "
350 		    "<pool> <vdev> ...\n"));
351 	case HELP_ATTACH:
352 		return (gettext("\tattach [-fsw] [-o property=value] "
353 		    "<pool> <device> <new-device>\n"));
354 	case HELP_CLEAR:
355 		return (gettext("\tclear [-nF] <pool> [device]\n"));
356 	case HELP_CREATE:
357 		return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
358 		    "\t    [-O file-system-property=value] ... \n"
359 		    "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
360 	case HELP_CHECKPOINT:
361 		return (gettext("\tcheckpoint [-d [-w]] <pool> ...\n"));
362 	case HELP_DESTROY:
363 		return (gettext("\tdestroy [-f] <pool>\n"));
364 	case HELP_DETACH:
365 		return (gettext("\tdetach <pool> <device>\n"));
366 	case HELP_EXPORT:
367 		return (gettext("\texport [-af] <pool> ...\n"));
368 	case HELP_HISTORY:
369 		return (gettext("\thistory [-il] [<pool>] ...\n"));
370 	case HELP_IMPORT:
371 		return (gettext("\timport [-d dir] [-D]\n"
372 		    "\timport [-o mntopts] [-o property=value] ... \n"
373 		    "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
374 		    "[-R root] [-F [-n]] -a\n"
375 		    "\timport [-o mntopts] [-o property=value] ... \n"
376 		    "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
377 		    "[-R root] [-F [-n]]\n"
378 		    "\t    [--rewind-to-checkpoint] <pool | id> [newpool]\n"));
379 	case HELP_IOSTAT:
380 		return (gettext("\tiostat [[[-c [script1,script2,...]"
381 		    "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
382 		    "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
383 		    " [[-n] interval [count]]\n"));
384 	case HELP_LABELCLEAR:
385 		return (gettext("\tlabelclear [-f] <vdev>\n"));
386 	case HELP_LIST:
387 		return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
388 		    "[-T d|u] [pool] ... \n"
389 		    "\t    [interval [count]]\n"));
390 	case HELP_OFFLINE:
391 		return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
392 	case HELP_ONLINE:
393 		return (gettext("\tonline [-e] <pool> <device> ...\n"));
394 	case HELP_REPLACE:
395 		return (gettext("\treplace [-fsw] [-o property=value] "
396 		    "<pool> <device> [new-device]\n"));
397 	case HELP_REMOVE:
398 		return (gettext("\tremove [-npsw] <pool> <device> ...\n"));
399 	case HELP_REOPEN:
400 		return (gettext("\treopen [-n] <pool>\n"));
401 	case HELP_INITIALIZE:
402 		return (gettext("\tinitialize [-c | -s] [-w] <pool> "
403 		    "[<device> ...]\n"));
404 	case HELP_SCRUB:
405 		return (gettext("\tscrub [-s | -p] [-w] <pool> ...\n"));
406 	case HELP_RESILVER:
407 		return (gettext("\tresilver <pool> ...\n"));
408 	case HELP_TRIM:
409 		return (gettext("\ttrim [-dw] [-r <rate>] [-c | -s] <pool> "
410 		    "[<device> ...]\n"));
411 	case HELP_STATUS:
412 		return (gettext("\tstatus [-c [script1,script2,...]] "
413 		    "[-igLpPstvxD]  [-T d|u] [pool] ... \n"
414 		    "\t    [interval [count]]\n"));
415 	case HELP_UPGRADE:
416 		return (gettext("\tupgrade\n"
417 		    "\tupgrade -v\n"
418 		    "\tupgrade [-V version] <-a | pool ...>\n"));
419 	case HELP_EVENTS:
420 		return (gettext("\tevents [-vHf [pool] | -c]\n"));
421 	case HELP_GET:
422 		return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
423 		    "<\"all\" | property[,...]> <pool> ...\n"));
424 	case HELP_SET:
425 		return (gettext("\tset <property=value> <pool> \n"));
426 	case HELP_SPLIT:
427 		return (gettext("\tsplit [-gLnPl] [-R altroot] [-o mntopts]\n"
428 		    "\t    [-o property=value] <pool> <newpool> "
429 		    "[<device> ...]\n"));
430 	case HELP_REGUID:
431 		return (gettext("\treguid <pool>\n"));
432 	case HELP_SYNC:
433 		return (gettext("\tsync [pool] ...\n"));
434 	case HELP_VERSION:
435 		return (gettext("\tversion\n"));
436 	case HELP_WAIT:
437 		return (gettext("\twait [-Hp] [-T d|u] [-t <activity>[,...]] "
438 		    "<pool> [interval]\n"));
439 	default:
440 		__builtin_unreachable();
441 	}
442 }
443 
444 static void
445 zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res)
446 {
447 	uint_t children = 0;
448 	nvlist_t **child;
449 	uint_t i;
450 
451 	(void) nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
452 	    &child, &children);
453 
454 	if (children == 0) {
455 		char *path = zpool_vdev_name(g_zfs, zhp, nvroot,
456 		    VDEV_NAME_PATH);
457 
458 		if (strcmp(path, VDEV_TYPE_INDIRECT) != 0 &&
459 		    strcmp(path, VDEV_TYPE_HOLE) != 0)
460 			fnvlist_add_boolean(res, path);
461 
462 		free(path);
463 		return;
464 	}
465 
466 	for (i = 0; i < children; i++) {
467 		zpool_collect_leaves(zhp, child[i], res);
468 	}
469 }
470 
471 /*
472  * Callback routine that will print out a pool property value.
473  */
474 static int
475 print_pool_prop_cb(int prop, void *cb)
476 {
477 	FILE *fp = cb;
478 
479 	(void) fprintf(fp, "\t%-19s  ", zpool_prop_to_name(prop));
480 
481 	if (zpool_prop_readonly(prop))
482 		(void) fprintf(fp, "  NO   ");
483 	else
484 		(void) fprintf(fp, " YES   ");
485 
486 	if (zpool_prop_values(prop) == NULL)
487 		(void) fprintf(fp, "-\n");
488 	else
489 		(void) fprintf(fp, "%s\n", zpool_prop_values(prop));
490 
491 	return (ZPROP_CONT);
492 }
493 
494 /*
495  * Callback routine that will print out a vdev property value.
496  */
497 static int
498 print_vdev_prop_cb(int prop, void *cb)
499 {
500 	FILE *fp = cb;
501 
502 	(void) fprintf(fp, "\t%-19s  ", vdev_prop_to_name(prop));
503 
504 	if (vdev_prop_readonly(prop))
505 		(void) fprintf(fp, "  NO   ");
506 	else
507 		(void) fprintf(fp, " YES   ");
508 
509 	if (vdev_prop_values(prop) == NULL)
510 		(void) fprintf(fp, "-\n");
511 	else
512 		(void) fprintf(fp, "%s\n", vdev_prop_values(prop));
513 
514 	return (ZPROP_CONT);
515 }
516 
517 /*
518  * Display usage message.  If we're inside a command, display only the usage for
519  * that command.  Otherwise, iterate over the entire command table and display
520  * a complete usage message.
521  */
522 static void
523 usage(boolean_t requested)
524 {
525 	FILE *fp = requested ? stdout : stderr;
526 
527 	if (current_command == NULL) {
528 		int i;
529 
530 		(void) fprintf(fp, gettext("usage: zpool command args ...\n"));
531 		(void) fprintf(fp,
532 		    gettext("where 'command' is one of the following:\n\n"));
533 
534 		for (i = 0; i < NCOMMAND; i++) {
535 			if (command_table[i].name == NULL)
536 				(void) fprintf(fp, "\n");
537 			else
538 				(void) fprintf(fp, "%s",
539 				    get_usage(command_table[i].usage));
540 		}
541 	} else {
542 		(void) fprintf(fp, gettext("usage:\n"));
543 		(void) fprintf(fp, "%s", get_usage(current_command->usage));
544 	}
545 
546 	if (current_command != NULL &&
547 	    current_prop_type != (ZFS_TYPE_POOL | ZFS_TYPE_VDEV) &&
548 	    ((strcmp(current_command->name, "set") == 0) ||
549 	    (strcmp(current_command->name, "get") == 0) ||
550 	    (strcmp(current_command->name, "list") == 0))) {
551 
552 		(void) fprintf(fp,
553 		    gettext("\nthe following properties are supported:\n"));
554 
555 		(void) fprintf(fp, "\n\t%-19s  %s   %s\n\n",
556 		    "PROPERTY", "EDIT", "VALUES");
557 
558 		/* Iterate over all properties */
559 		if (current_prop_type == ZFS_TYPE_POOL) {
560 			(void) zprop_iter(print_pool_prop_cb, fp, B_FALSE,
561 			    B_TRUE, current_prop_type);
562 
563 			(void) fprintf(fp, "\t%-19s   ", "feature@...");
564 			(void) fprintf(fp, "YES   "
565 			    "disabled | enabled | active\n");
566 
567 			(void) fprintf(fp, gettext("\nThe feature@ properties "
568 			    "must be appended with a feature name.\n"
569 			    "See zpool-features(7).\n"));
570 		} else if (current_prop_type == ZFS_TYPE_VDEV) {
571 			(void) zprop_iter(print_vdev_prop_cb, fp, B_FALSE,
572 			    B_TRUE, current_prop_type);
573 		}
574 	}
575 
576 	/*
577 	 * See comments at end of main().
578 	 */
579 	if (getenv("ZFS_ABORT") != NULL) {
580 		(void) printf("dumping core by request\n");
581 		abort();
582 	}
583 
584 	exit(requested ? 0 : 2);
585 }
586 
587 /*
588  * zpool initialize [-c | -s] [-w] <pool> [<vdev> ...]
589  * Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
590  * if none specified.
591  *
592  *	-c	Cancel. Ends active initializing.
593  *	-s	Suspend. Initializing can then be restarted with no flags.
594  *	-w	Wait. Blocks until initializing has completed.
595  */
596 int
597 zpool_do_initialize(int argc, char **argv)
598 {
599 	int c;
600 	char *poolname;
601 	zpool_handle_t *zhp;
602 	nvlist_t *vdevs;
603 	int err = 0;
604 	boolean_t wait = B_FALSE;
605 
606 	struct option long_options[] = {
607 		{"cancel",	no_argument,		NULL, 'c'},
608 		{"suspend",	no_argument,		NULL, 's'},
609 		{"wait",	no_argument,		NULL, 'w'},
610 		{0, 0, 0, 0}
611 	};
612 
613 	pool_initialize_func_t cmd_type = POOL_INITIALIZE_START;
614 	while ((c = getopt_long(argc, argv, "csw", long_options, NULL)) != -1) {
615 		switch (c) {
616 		case 'c':
617 			if (cmd_type != POOL_INITIALIZE_START &&
618 			    cmd_type != POOL_INITIALIZE_CANCEL) {
619 				(void) fprintf(stderr, gettext("-c cannot be "
620 				    "combined with other options\n"));
621 				usage(B_FALSE);
622 			}
623 			cmd_type = POOL_INITIALIZE_CANCEL;
624 			break;
625 		case 's':
626 			if (cmd_type != POOL_INITIALIZE_START &&
627 			    cmd_type != POOL_INITIALIZE_SUSPEND) {
628 				(void) fprintf(stderr, gettext("-s cannot be "
629 				    "combined with other options\n"));
630 				usage(B_FALSE);
631 			}
632 			cmd_type = POOL_INITIALIZE_SUSPEND;
633 			break;
634 		case 'w':
635 			wait = B_TRUE;
636 			break;
637 		case '?':
638 			if (optopt != 0) {
639 				(void) fprintf(stderr,
640 				    gettext("invalid option '%c'\n"), optopt);
641 			} else {
642 				(void) fprintf(stderr,
643 				    gettext("invalid option '%s'\n"),
644 				    argv[optind - 1]);
645 			}
646 			usage(B_FALSE);
647 		}
648 	}
649 
650 	argc -= optind;
651 	argv += optind;
652 
653 	if (argc < 1) {
654 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
655 		usage(B_FALSE);
656 		return (-1);
657 	}
658 
659 	if (wait && (cmd_type != POOL_INITIALIZE_START)) {
660 		(void) fprintf(stderr, gettext("-w cannot be used with -c or "
661 		    "-s\n"));
662 		usage(B_FALSE);
663 	}
664 
665 	poolname = argv[0];
666 	zhp = zpool_open(g_zfs, poolname);
667 	if (zhp == NULL)
668 		return (-1);
669 
670 	vdevs = fnvlist_alloc();
671 	if (argc == 1) {
672 		/* no individual leaf vdevs specified, so add them all */
673 		nvlist_t *config = zpool_get_config(zhp, NULL);
674 		nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
675 		    ZPOOL_CONFIG_VDEV_TREE);
676 		zpool_collect_leaves(zhp, nvroot, vdevs);
677 	} else {
678 		for (int i = 1; i < argc; i++) {
679 			fnvlist_add_boolean(vdevs, argv[i]);
680 		}
681 	}
682 
683 	if (wait)
684 		err = zpool_initialize_wait(zhp, cmd_type, vdevs);
685 	else
686 		err = zpool_initialize(zhp, cmd_type, vdevs);
687 
688 	fnvlist_free(vdevs);
689 	zpool_close(zhp);
690 
691 	return (err);
692 }
693 
694 /*
695  * print a pool vdev config for dry runs
696  */
697 static void
698 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
699     const char *match, int name_flags)
700 {
701 	nvlist_t **child;
702 	uint_t c, children;
703 	char *vname;
704 	boolean_t printed = B_FALSE;
705 
706 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
707 	    &child, &children) != 0) {
708 		if (name != NULL)
709 			(void) printf("\t%*s%s\n", indent, "", name);
710 		return;
711 	}
712 
713 	for (c = 0; c < children; c++) {
714 		uint64_t is_log = B_FALSE, is_hole = B_FALSE;
715 		char *class = "";
716 
717 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
718 		    &is_hole);
719 
720 		if (is_hole == B_TRUE) {
721 			continue;
722 		}
723 
724 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
725 		    &is_log);
726 		if (is_log)
727 			class = VDEV_ALLOC_BIAS_LOG;
728 		(void) nvlist_lookup_string(child[c],
729 		    ZPOOL_CONFIG_ALLOCATION_BIAS, &class);
730 		if (strcmp(match, class) != 0)
731 			continue;
732 
733 		if (!printed && name != NULL) {
734 			(void) printf("\t%*s%s\n", indent, "", name);
735 			printed = B_TRUE;
736 		}
737 		vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
738 		print_vdev_tree(zhp, vname, child[c], indent + 2, "",
739 		    name_flags);
740 		free(vname);
741 	}
742 }
743 
744 /*
745  * Print the list of l2cache devices for dry runs.
746  */
747 static void
748 print_cache_list(nvlist_t *nv, int indent)
749 {
750 	nvlist_t **child;
751 	uint_t c, children;
752 
753 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
754 	    &child, &children) == 0 && children > 0) {
755 		(void) printf("\t%*s%s\n", indent, "", "cache");
756 	} else {
757 		return;
758 	}
759 	for (c = 0; c < children; c++) {
760 		char *vname;
761 
762 		vname = zpool_vdev_name(g_zfs, NULL, child[c], 0);
763 		(void) printf("\t%*s%s\n", indent + 2, "", vname);
764 		free(vname);
765 	}
766 }
767 
768 /*
769  * Print the list of spares for dry runs.
770  */
771 static void
772 print_spare_list(nvlist_t *nv, int indent)
773 {
774 	nvlist_t **child;
775 	uint_t c, children;
776 
777 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
778 	    &child, &children) == 0 && children > 0) {
779 		(void) printf("\t%*s%s\n", indent, "", "spares");
780 	} else {
781 		return;
782 	}
783 	for (c = 0; c < children; c++) {
784 		char *vname;
785 
786 		vname = zpool_vdev_name(g_zfs, NULL, child[c], 0);
787 		(void) printf("\t%*s%s\n", indent + 2, "", vname);
788 		free(vname);
789 	}
790 }
791 
792 static boolean_t
793 prop_list_contains_feature(nvlist_t *proplist)
794 {
795 	nvpair_t *nvp;
796 	for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
797 	    nvp = nvlist_next_nvpair(proplist, nvp)) {
798 		if (zpool_prop_feature(nvpair_name(nvp)))
799 			return (B_TRUE);
800 	}
801 	return (B_FALSE);
802 }
803 
804 /*
805  * Add a property pair (name, string-value) into a property nvlist.
806  */
807 static int
808 add_prop_list(const char *propname, char *propval, nvlist_t **props,
809     boolean_t poolprop)
810 {
811 	zpool_prop_t prop = ZPOOL_PROP_INVAL;
812 	nvlist_t *proplist;
813 	const char *normnm;
814 	char *strval;
815 
816 	if (*props == NULL &&
817 	    nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
818 		(void) fprintf(stderr,
819 		    gettext("internal error: out of memory\n"));
820 		return (1);
821 	}
822 
823 	proplist = *props;
824 
825 	if (poolprop) {
826 		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
827 		const char *cname =
828 		    zpool_prop_to_name(ZPOOL_PROP_COMPATIBILITY);
829 
830 		if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
831 		    (!zpool_prop_feature(propname) &&
832 		    !zpool_prop_vdev(propname))) {
833 			(void) fprintf(stderr, gettext("property '%s' is "
834 			    "not a valid pool or vdev property\n"), propname);
835 			return (2);
836 		}
837 
838 		/*
839 		 * feature@ properties and version should not be specified
840 		 * at the same time.
841 		 */
842 		if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
843 		    nvlist_exists(proplist, vname)) ||
844 		    (prop == ZPOOL_PROP_VERSION &&
845 		    prop_list_contains_feature(proplist))) {
846 			(void) fprintf(stderr, gettext("'feature@' and "
847 			    "'version' properties cannot be specified "
848 			    "together\n"));
849 			return (2);
850 		}
851 
852 		/*
853 		 * if version is specified, only "legacy" compatibility
854 		 * may be requested
855 		 */
856 		if ((prop == ZPOOL_PROP_COMPATIBILITY &&
857 		    strcmp(propval, ZPOOL_COMPAT_LEGACY) != 0 &&
858 		    nvlist_exists(proplist, vname)) ||
859 		    (prop == ZPOOL_PROP_VERSION &&
860 		    nvlist_exists(proplist, cname) &&
861 		    strcmp(fnvlist_lookup_string(proplist, cname),
862 		    ZPOOL_COMPAT_LEGACY) != 0)) {
863 			(void) fprintf(stderr, gettext("when 'version' is "
864 			    "specified, the 'compatibility' feature may only "
865 			    "be set to '" ZPOOL_COMPAT_LEGACY "'\n"));
866 			return (2);
867 		}
868 
869 		if (zpool_prop_feature(propname) || zpool_prop_vdev(propname))
870 			normnm = propname;
871 		else
872 			normnm = zpool_prop_to_name(prop);
873 	} else {
874 		zfs_prop_t fsprop = zfs_name_to_prop(propname);
875 
876 		if (zfs_prop_valid_for_type(fsprop, ZFS_TYPE_FILESYSTEM,
877 		    B_FALSE)) {
878 			normnm = zfs_prop_to_name(fsprop);
879 		} else if (zfs_prop_user(propname) ||
880 		    zfs_prop_userquota(propname)) {
881 			normnm = propname;
882 		} else {
883 			(void) fprintf(stderr, gettext("property '%s' is "
884 			    "not a valid filesystem property\n"), propname);
885 			return (2);
886 		}
887 	}
888 
889 	if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
890 	    prop != ZPOOL_PROP_CACHEFILE) {
891 		(void) fprintf(stderr, gettext("property '%s' "
892 		    "specified multiple times\n"), propname);
893 		return (2);
894 	}
895 
896 	if (nvlist_add_string(proplist, normnm, propval) != 0) {
897 		(void) fprintf(stderr, gettext("internal "
898 		    "error: out of memory\n"));
899 		return (1);
900 	}
901 
902 	return (0);
903 }
904 
905 /*
906  * Set a default property pair (name, string-value) in a property nvlist
907  */
908 static int
909 add_prop_list_default(const char *propname, char *propval, nvlist_t **props,
910     boolean_t poolprop)
911 {
912 	char *pval;
913 
914 	if (nvlist_lookup_string(*props, propname, &pval) == 0)
915 		return (0);
916 
917 	return (add_prop_list(propname, propval, props, B_TRUE));
918 }
919 
920 /*
921  * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
922  *
923  *	-f	Force addition of devices, even if they appear in use
924  *	-g	Display guid for individual vdev name.
925  *	-L	Follow links when resolving vdev path name.
926  *	-n	Do not add the devices, but display the resulting layout if
927  *		they were to be added.
928  *	-o	Set property=value.
929  *	-P	Display full path for vdev name.
930  *
931  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
932  * handled by make_root_vdev(), which constructs the nvlist needed to pass to
933  * libzfs.
934  */
935 int
936 zpool_do_add(int argc, char **argv)
937 {
938 	boolean_t force = B_FALSE;
939 	boolean_t dryrun = B_FALSE;
940 	int name_flags = 0;
941 	int c;
942 	nvlist_t *nvroot;
943 	char *poolname;
944 	int ret;
945 	zpool_handle_t *zhp;
946 	nvlist_t *config;
947 	nvlist_t *props = NULL;
948 	char *propval;
949 
950 	/* check options */
951 	while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
952 		switch (c) {
953 		case 'f':
954 			force = B_TRUE;
955 			break;
956 		case 'g':
957 			name_flags |= VDEV_NAME_GUID;
958 			break;
959 		case 'L':
960 			name_flags |= VDEV_NAME_FOLLOW_LINKS;
961 			break;
962 		case 'n':
963 			dryrun = B_TRUE;
964 			break;
965 		case 'o':
966 			if ((propval = strchr(optarg, '=')) == NULL) {
967 				(void) fprintf(stderr, gettext("missing "
968 				    "'=' for -o option\n"));
969 				usage(B_FALSE);
970 			}
971 			*propval = '\0';
972 			propval++;
973 
974 			if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
975 			    (add_prop_list(optarg, propval, &props, B_TRUE)))
976 				usage(B_FALSE);
977 			break;
978 		case 'P':
979 			name_flags |= VDEV_NAME_PATH;
980 			break;
981 		case '?':
982 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
983 			    optopt);
984 			usage(B_FALSE);
985 		}
986 	}
987 
988 	argc -= optind;
989 	argv += optind;
990 
991 	/* get pool name and check number of arguments */
992 	if (argc < 1) {
993 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
994 		usage(B_FALSE);
995 	}
996 	if (argc < 2) {
997 		(void) fprintf(stderr, gettext("missing vdev specification\n"));
998 		usage(B_FALSE);
999 	}
1000 
1001 	poolname = argv[0];
1002 
1003 	argc--;
1004 	argv++;
1005 
1006 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
1007 		return (1);
1008 
1009 	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
1010 		(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
1011 		    poolname);
1012 		zpool_close(zhp);
1013 		return (1);
1014 	}
1015 
1016 	/* unless manually specified use "ashift" pool property (if set) */
1017 	if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
1018 		int intval;
1019 		zprop_source_t src;
1020 		char strval[ZPOOL_MAXPROPLEN];
1021 
1022 		intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
1023 		if (src != ZPROP_SRC_DEFAULT) {
1024 			(void) sprintf(strval, "%" PRId32, intval);
1025 			verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
1026 			    &props, B_TRUE) == 0);
1027 		}
1028 	}
1029 
1030 	/* pass off to make_root_vdev for processing */
1031 	nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
1032 	    argc, argv);
1033 	if (nvroot == NULL) {
1034 		zpool_close(zhp);
1035 		return (1);
1036 	}
1037 
1038 	if (dryrun) {
1039 		nvlist_t *poolnvroot;
1040 		nvlist_t **l2child, **sparechild;
1041 		uint_t l2children, sparechildren, c;
1042 		char *vname;
1043 		boolean_t hadcache = B_FALSE, hadspare = B_FALSE;
1044 
1045 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1046 		    &poolnvroot) == 0);
1047 
1048 		(void) printf(gettext("would update '%s' to the following "
1049 		    "configuration:\n\n"), zpool_get_name(zhp));
1050 
1051 		/* print original main pool and new tree */
1052 		print_vdev_tree(zhp, poolname, poolnvroot, 0, "",
1053 		    name_flags | VDEV_NAME_TYPE_ID);
1054 		print_vdev_tree(zhp, NULL, nvroot, 0, "", name_flags);
1055 
1056 		/* print other classes: 'dedup', 'special', and 'log' */
1057 		if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_DEDUP)) {
1058 			print_vdev_tree(zhp, "dedup", poolnvroot, 0,
1059 			    VDEV_ALLOC_BIAS_DEDUP, name_flags);
1060 			print_vdev_tree(zhp, NULL, nvroot, 0,
1061 			    VDEV_ALLOC_BIAS_DEDUP, name_flags);
1062 		} else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_DEDUP)) {
1063 			print_vdev_tree(zhp, "dedup", nvroot, 0,
1064 			    VDEV_ALLOC_BIAS_DEDUP, name_flags);
1065 		}
1066 
1067 		if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
1068 			print_vdev_tree(zhp, "special", poolnvroot, 0,
1069 			    VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1070 			print_vdev_tree(zhp, NULL, nvroot, 0,
1071 			    VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1072 		} else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
1073 			print_vdev_tree(zhp, "special", nvroot, 0,
1074 			    VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1075 		}
1076 
1077 		if (num_logs(poolnvroot) > 0) {
1078 			print_vdev_tree(zhp, "logs", poolnvroot, 0,
1079 			    VDEV_ALLOC_BIAS_LOG, name_flags);
1080 			print_vdev_tree(zhp, NULL, nvroot, 0,
1081 			    VDEV_ALLOC_BIAS_LOG, name_flags);
1082 		} else if (num_logs(nvroot) > 0) {
1083 			print_vdev_tree(zhp, "logs", nvroot, 0,
1084 			    VDEV_ALLOC_BIAS_LOG, name_flags);
1085 		}
1086 
1087 		/* Do the same for the caches */
1088 		if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_L2CACHE,
1089 		    &l2child, &l2children) == 0 && l2children) {
1090 			hadcache = B_TRUE;
1091 			(void) printf(gettext("\tcache\n"));
1092 			for (c = 0; c < l2children; c++) {
1093 				vname = zpool_vdev_name(g_zfs, NULL,
1094 				    l2child[c], name_flags);
1095 				(void) printf("\t  %s\n", vname);
1096 				free(vname);
1097 			}
1098 		}
1099 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
1100 		    &l2child, &l2children) == 0 && l2children) {
1101 			if (!hadcache)
1102 				(void) printf(gettext("\tcache\n"));
1103 			for (c = 0; c < l2children; c++) {
1104 				vname = zpool_vdev_name(g_zfs, NULL,
1105 				    l2child[c], name_flags);
1106 				(void) printf("\t  %s\n", vname);
1107 				free(vname);
1108 			}
1109 		}
1110 		/* And finally the spares */
1111 		if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_SPARES,
1112 		    &sparechild, &sparechildren) == 0 && sparechildren > 0) {
1113 			hadspare = B_TRUE;
1114 			(void) printf(gettext("\tspares\n"));
1115 			for (c = 0; c < sparechildren; c++) {
1116 				vname = zpool_vdev_name(g_zfs, NULL,
1117 				    sparechild[c], name_flags);
1118 				(void) printf("\t  %s\n", vname);
1119 				free(vname);
1120 			}
1121 		}
1122 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
1123 		    &sparechild, &sparechildren) == 0 && sparechildren > 0) {
1124 			if (!hadspare)
1125 				(void) printf(gettext("\tspares\n"));
1126 			for (c = 0; c < sparechildren; c++) {
1127 				vname = zpool_vdev_name(g_zfs, NULL,
1128 				    sparechild[c], name_flags);
1129 				(void) printf("\t  %s\n", vname);
1130 				free(vname);
1131 			}
1132 		}
1133 
1134 		ret = 0;
1135 	} else {
1136 		ret = (zpool_add(zhp, nvroot) != 0);
1137 	}
1138 
1139 	nvlist_free(props);
1140 	nvlist_free(nvroot);
1141 	zpool_close(zhp);
1142 
1143 	return (ret);
1144 }
1145 
1146 /*
1147  * zpool remove [-npsw] <pool> <vdev> ...
1148  *
1149  * Removes the given vdev from the pool.
1150  */
1151 int
1152 zpool_do_remove(int argc, char **argv)
1153 {
1154 	char *poolname;
1155 	int i, ret = 0;
1156 	zpool_handle_t *zhp = NULL;
1157 	boolean_t stop = B_FALSE;
1158 	int c;
1159 	boolean_t noop = B_FALSE;
1160 	boolean_t parsable = B_FALSE;
1161 	boolean_t wait = B_FALSE;
1162 
1163 	/* check options */
1164 	while ((c = getopt(argc, argv, "npsw")) != -1) {
1165 		switch (c) {
1166 		case 'n':
1167 			noop = B_TRUE;
1168 			break;
1169 		case 'p':
1170 			parsable = B_TRUE;
1171 			break;
1172 		case 's':
1173 			stop = B_TRUE;
1174 			break;
1175 		case 'w':
1176 			wait = B_TRUE;
1177 			break;
1178 		case '?':
1179 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1180 			    optopt);
1181 			usage(B_FALSE);
1182 		}
1183 	}
1184 
1185 	argc -= optind;
1186 	argv += optind;
1187 
1188 	/* get pool name and check number of arguments */
1189 	if (argc < 1) {
1190 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
1191 		usage(B_FALSE);
1192 	}
1193 
1194 	poolname = argv[0];
1195 
1196 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
1197 		return (1);
1198 
1199 	if (stop && noop) {
1200 		(void) fprintf(stderr, gettext("stop request ignored\n"));
1201 		return (0);
1202 	}
1203 
1204 	if (stop) {
1205 		if (argc > 1) {
1206 			(void) fprintf(stderr, gettext("too many arguments\n"));
1207 			usage(B_FALSE);
1208 		}
1209 		if (zpool_vdev_remove_cancel(zhp) != 0)
1210 			ret = 1;
1211 		if (wait) {
1212 			(void) fprintf(stderr, gettext("invalid option "
1213 			    "combination: -w cannot be used with -s\n"));
1214 			usage(B_FALSE);
1215 		}
1216 	} else {
1217 		if (argc < 2) {
1218 			(void) fprintf(stderr, gettext("missing device\n"));
1219 			usage(B_FALSE);
1220 		}
1221 
1222 		for (i = 1; i < argc; i++) {
1223 			if (noop) {
1224 				uint64_t size;
1225 
1226 				if (zpool_vdev_indirect_size(zhp, argv[i],
1227 				    &size) != 0) {
1228 					ret = 1;
1229 					break;
1230 				}
1231 				if (parsable) {
1232 					(void) printf("%s %llu\n",
1233 					    argv[i], (unsigned long long)size);
1234 				} else {
1235 					char valstr[32];
1236 					zfs_nicenum(size, valstr,
1237 					    sizeof (valstr));
1238 					(void) printf("Memory that will be "
1239 					    "used after removing %s: %s\n",
1240 					    argv[i], valstr);
1241 				}
1242 			} else {
1243 				if (zpool_vdev_remove(zhp, argv[i]) != 0)
1244 					ret = 1;
1245 			}
1246 		}
1247 
1248 		if (ret == 0 && wait)
1249 			ret = zpool_wait(zhp, ZPOOL_WAIT_REMOVE);
1250 	}
1251 	zpool_close(zhp);
1252 
1253 	return (ret);
1254 }
1255 
1256 /*
1257  * Return 1 if a vdev is active (being used in a pool)
1258  * Return 0 if a vdev is inactive (offlined or faulted, or not in active pool)
1259  *
1260  * This is useful for checking if a disk in an active pool is offlined or
1261  * faulted.
1262  */
1263 static int
1264 vdev_is_active(char *vdev_path)
1265 {
1266 	int fd;
1267 	fd = open(vdev_path, O_EXCL);
1268 	if (fd < 0) {
1269 		return (1);   /* cant open O_EXCL - disk is active */
1270 	}
1271 
1272 	close(fd);
1273 	return (0);   /* disk is inactive in the pool */
1274 }
1275 
1276 /*
1277  * zpool labelclear [-f] <vdev>
1278  *
1279  *	-f	Force clearing the label for the vdevs which are members of
1280  *		the exported or foreign pools.
1281  *
1282  * Verifies that the vdev is not active and zeros out the label information
1283  * on the device.
1284  */
1285 int
1286 zpool_do_labelclear(int argc, char **argv)
1287 {
1288 	char vdev[MAXPATHLEN];
1289 	char *name = NULL;
1290 	struct stat st;
1291 	int c, fd = -1, ret = 0;
1292 	nvlist_t *config;
1293 	pool_state_t state;
1294 	boolean_t inuse = B_FALSE;
1295 	boolean_t force = B_FALSE;
1296 
1297 	/* check options */
1298 	while ((c = getopt(argc, argv, "f")) != -1) {
1299 		switch (c) {
1300 		case 'f':
1301 			force = B_TRUE;
1302 			break;
1303 		default:
1304 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1305 			    optopt);
1306 			usage(B_FALSE);
1307 		}
1308 	}
1309 
1310 	argc -= optind;
1311 	argv += optind;
1312 
1313 	/* get vdev name */
1314 	if (argc < 1) {
1315 		(void) fprintf(stderr, gettext("missing vdev name\n"));
1316 		usage(B_FALSE);
1317 	}
1318 	if (argc > 1) {
1319 		(void) fprintf(stderr, gettext("too many arguments\n"));
1320 		usage(B_FALSE);
1321 	}
1322 
1323 	/*
1324 	 * Check if we were given absolute path and use it as is.
1325 	 * Otherwise if the provided vdev name doesn't point to a file,
1326 	 * try prepending expected disk paths and partition numbers.
1327 	 */
1328 	(void) strlcpy(vdev, argv[0], sizeof (vdev));
1329 	if (vdev[0] != '/' && stat(vdev, &st) != 0) {
1330 		int error;
1331 
1332 		error = zfs_resolve_shortname(argv[0], vdev, MAXPATHLEN);
1333 		if (error == 0 && zfs_dev_is_whole_disk(vdev)) {
1334 			if (zfs_append_partition(vdev, MAXPATHLEN) == -1)
1335 				error = ENOENT;
1336 		}
1337 
1338 		if (error || (stat(vdev, &st) != 0)) {
1339 			(void) fprintf(stderr, gettext(
1340 			    "failed to find device %s, try specifying absolute "
1341 			    "path instead\n"), argv[0]);
1342 			return (1);
1343 		}
1344 	}
1345 
1346 	if ((fd = open(vdev, O_RDWR)) < 0) {
1347 		(void) fprintf(stderr, gettext("failed to open %s: %s\n"),
1348 		    vdev, strerror(errno));
1349 		return (1);
1350 	}
1351 
1352 	/*
1353 	 * Flush all dirty pages for the block device.  This should not be
1354 	 * fatal when the device does not support BLKFLSBUF as would be the
1355 	 * case for a file vdev.
1356 	 */
1357 	if ((zfs_dev_flush(fd) != 0) && (errno != ENOTTY))
1358 		(void) fprintf(stderr, gettext("failed to invalidate "
1359 		    "cache for %s: %s\n"), vdev, strerror(errno));
1360 
1361 	if (zpool_read_label(fd, &config, NULL) != 0) {
1362 		(void) fprintf(stderr,
1363 		    gettext("failed to read label from %s\n"), vdev);
1364 		ret = 1;
1365 		goto errout;
1366 	}
1367 	nvlist_free(config);
1368 
1369 	ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
1370 	if (ret != 0) {
1371 		(void) fprintf(stderr,
1372 		    gettext("failed to check state for %s\n"), vdev);
1373 		ret = 1;
1374 		goto errout;
1375 	}
1376 
1377 	if (!inuse)
1378 		goto wipe_label;
1379 
1380 	switch (state) {
1381 	default:
1382 	case POOL_STATE_ACTIVE:
1383 	case POOL_STATE_SPARE:
1384 	case POOL_STATE_L2CACHE:
1385 		/*
1386 		 * We allow the user to call 'zpool offline -f'
1387 		 * on an offlined disk in an active pool. We can check if
1388 		 * the disk is online by calling vdev_is_active().
1389 		 */
1390 		if (force && !vdev_is_active(vdev))
1391 			break;
1392 
1393 		(void) fprintf(stderr, gettext(
1394 		    "%s is a member (%s) of pool \"%s\""),
1395 		    vdev, zpool_pool_state_to_name(state), name);
1396 
1397 		if (force) {
1398 			(void) fprintf(stderr, gettext(
1399 			    ". Offline the disk first to clear its label."));
1400 		}
1401 		printf("\n");
1402 		ret = 1;
1403 		goto errout;
1404 
1405 	case POOL_STATE_EXPORTED:
1406 		if (force)
1407 			break;
1408 		(void) fprintf(stderr, gettext(
1409 		    "use '-f' to override the following error:\n"
1410 		    "%s is a member of exported pool \"%s\"\n"),
1411 		    vdev, name);
1412 		ret = 1;
1413 		goto errout;
1414 
1415 	case POOL_STATE_POTENTIALLY_ACTIVE:
1416 		if (force)
1417 			break;
1418 		(void) fprintf(stderr, gettext(
1419 		    "use '-f' to override the following error:\n"
1420 		    "%s is a member of potentially active pool \"%s\"\n"),
1421 		    vdev, name);
1422 		ret = 1;
1423 		goto errout;
1424 
1425 	case POOL_STATE_DESTROYED:
1426 		/* inuse should never be set for a destroyed pool */
1427 		assert(0);
1428 		break;
1429 	}
1430 
1431 wipe_label:
1432 	ret = zpool_clear_label(fd);
1433 	if (ret != 0) {
1434 		(void) fprintf(stderr,
1435 		    gettext("failed to clear label for %s\n"), vdev);
1436 	}
1437 
1438 errout:
1439 	free(name);
1440 	(void) close(fd);
1441 
1442 	return (ret);
1443 }
1444 
1445 /*
1446  * zpool create [-fnd] [-o property=value] ...
1447  *		[-O file-system-property=value] ...
1448  *		[-R root] [-m mountpoint] <pool> <dev> ...
1449  *
1450  *	-f	Force creation, even if devices appear in use
1451  *	-n	Do not create the pool, but display the resulting layout if it
1452  *		were to be created.
1453  *      -R	Create a pool under an alternate root
1454  *      -m	Set default mountpoint for the root dataset.  By default it's
1455  *		'/<pool>'
1456  *	-o	Set property=value.
1457  *	-o	Set feature@feature=enabled|disabled.
1458  *	-d	Don't automatically enable all supported pool features
1459  *		(individual features can be enabled with -o).
1460  *	-O	Set fsproperty=value in the pool's root file system
1461  *
1462  * Creates the named pool according to the given vdev specification.  The
1463  * bulk of the vdev processing is done in make_root_vdev() in zpool_vdev.c.
1464  * Once we get the nvlist back from make_root_vdev(), we either print out the
1465  * contents (if '-n' was specified), or pass it to libzfs to do the creation.
1466  */
1467 int
1468 zpool_do_create(int argc, char **argv)
1469 {
1470 	boolean_t force = B_FALSE;
1471 	boolean_t dryrun = B_FALSE;
1472 	boolean_t enable_pool_features = B_TRUE;
1473 
1474 	int c;
1475 	nvlist_t *nvroot = NULL;
1476 	char *poolname;
1477 	char *tname = NULL;
1478 	int ret = 1;
1479 	char *altroot = NULL;
1480 	char *compat = NULL;
1481 	char *mountpoint = NULL;
1482 	nvlist_t *fsprops = NULL;
1483 	nvlist_t *props = NULL;
1484 	char *propval;
1485 
1486 	/* check options */
1487 	while ((c = getopt(argc, argv, ":fndR:m:o:O:t:")) != -1) {
1488 		switch (c) {
1489 		case 'f':
1490 			force = B_TRUE;
1491 			break;
1492 		case 'n':
1493 			dryrun = B_TRUE;
1494 			break;
1495 		case 'd':
1496 			enable_pool_features = B_FALSE;
1497 			break;
1498 		case 'R':
1499 			altroot = optarg;
1500 			if (add_prop_list(zpool_prop_to_name(
1501 			    ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1502 				goto errout;
1503 			if (add_prop_list_default(zpool_prop_to_name(
1504 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1505 				goto errout;
1506 			break;
1507 		case 'm':
1508 			/* Equivalent to -O mountpoint=optarg */
1509 			mountpoint = optarg;
1510 			break;
1511 		case 'o':
1512 			if ((propval = strchr(optarg, '=')) == NULL) {
1513 				(void) fprintf(stderr, gettext("missing "
1514 				    "'=' for -o option\n"));
1515 				goto errout;
1516 			}
1517 			*propval = '\0';
1518 			propval++;
1519 
1520 			if (add_prop_list(optarg, propval, &props, B_TRUE))
1521 				goto errout;
1522 
1523 			/*
1524 			 * If the user is creating a pool that doesn't support
1525 			 * feature flags, don't enable any features.
1526 			 */
1527 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1528 				char *end;
1529 				u_longlong_t ver;
1530 
1531 				ver = strtoull(propval, &end, 10);
1532 				if (*end == '\0' &&
1533 				    ver < SPA_VERSION_FEATURES) {
1534 					enable_pool_features = B_FALSE;
1535 				}
1536 			}
1537 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1538 				altroot = propval;
1539 			if (zpool_name_to_prop(optarg) ==
1540 			    ZPOOL_PROP_COMPATIBILITY)
1541 				compat = propval;
1542 			break;
1543 		case 'O':
1544 			if ((propval = strchr(optarg, '=')) == NULL) {
1545 				(void) fprintf(stderr, gettext("missing "
1546 				    "'=' for -O option\n"));
1547 				goto errout;
1548 			}
1549 			*propval = '\0';
1550 			propval++;
1551 
1552 			/*
1553 			 * Mountpoints are checked and then added later.
1554 			 * Uniquely among properties, they can be specified
1555 			 * more than once, to avoid conflict with -m.
1556 			 */
1557 			if (0 == strcmp(optarg,
1558 			    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1559 				mountpoint = propval;
1560 			} else if (add_prop_list(optarg, propval, &fsprops,
1561 			    B_FALSE)) {
1562 				goto errout;
1563 			}
1564 			break;
1565 		case 't':
1566 			/*
1567 			 * Sanity check temporary pool name.
1568 			 */
1569 			if (strchr(optarg, '/') != NULL) {
1570 				(void) fprintf(stderr, gettext("cannot create "
1571 				    "'%s': invalid character '/' in temporary "
1572 				    "name\n"), optarg);
1573 				(void) fprintf(stderr, gettext("use 'zfs "
1574 				    "create' to create a dataset\n"));
1575 				goto errout;
1576 			}
1577 
1578 			if (add_prop_list(zpool_prop_to_name(
1579 			    ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1580 				goto errout;
1581 			if (add_prop_list_default(zpool_prop_to_name(
1582 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1583 				goto errout;
1584 			tname = optarg;
1585 			break;
1586 		case ':':
1587 			(void) fprintf(stderr, gettext("missing argument for "
1588 			    "'%c' option\n"), optopt);
1589 			goto badusage;
1590 		case '?':
1591 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1592 			    optopt);
1593 			goto badusage;
1594 		}
1595 	}
1596 
1597 	argc -= optind;
1598 	argv += optind;
1599 
1600 	/* get pool name and check number of arguments */
1601 	if (argc < 1) {
1602 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
1603 		goto badusage;
1604 	}
1605 	if (argc < 2) {
1606 		(void) fprintf(stderr, gettext("missing vdev specification\n"));
1607 		goto badusage;
1608 	}
1609 
1610 	poolname = argv[0];
1611 
1612 	/*
1613 	 * As a special case, check for use of '/' in the name, and direct the
1614 	 * user to use 'zfs create' instead.
1615 	 */
1616 	if (strchr(poolname, '/') != NULL) {
1617 		(void) fprintf(stderr, gettext("cannot create '%s': invalid "
1618 		    "character '/' in pool name\n"), poolname);
1619 		(void) fprintf(stderr, gettext("use 'zfs create' to "
1620 		    "create a dataset\n"));
1621 		goto errout;
1622 	}
1623 
1624 	/* pass off to make_root_vdev for bulk processing */
1625 	nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1626 	    argc - 1, argv + 1);
1627 	if (nvroot == NULL)
1628 		goto errout;
1629 
1630 	/* make_root_vdev() allows 0 toplevel children if there are spares */
1631 	if (!zfs_allocatable_devs(nvroot)) {
1632 		(void) fprintf(stderr, gettext("invalid vdev "
1633 		    "specification: at least one toplevel vdev must be "
1634 		    "specified\n"));
1635 		goto errout;
1636 	}
1637 
1638 	if (altroot != NULL && altroot[0] != '/') {
1639 		(void) fprintf(stderr, gettext("invalid alternate root '%s': "
1640 		    "must be an absolute path\n"), altroot);
1641 		goto errout;
1642 	}
1643 
1644 	/*
1645 	 * Check the validity of the mountpoint and direct the user to use the
1646 	 * '-m' mountpoint option if it looks like its in use.
1647 	 */
1648 	if (mountpoint == NULL ||
1649 	    (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1650 	    strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1651 		char buf[MAXPATHLEN];
1652 		DIR *dirp;
1653 
1654 		if (mountpoint && mountpoint[0] != '/') {
1655 			(void) fprintf(stderr, gettext("invalid mountpoint "
1656 			    "'%s': must be an absolute path, 'legacy', or "
1657 			    "'none'\n"), mountpoint);
1658 			goto errout;
1659 		}
1660 
1661 		if (mountpoint == NULL) {
1662 			if (altroot != NULL)
1663 				(void) snprintf(buf, sizeof (buf), "%s/%s",
1664 				    altroot, poolname);
1665 			else
1666 				(void) snprintf(buf, sizeof (buf), "/%s",
1667 				    poolname);
1668 		} else {
1669 			if (altroot != NULL)
1670 				(void) snprintf(buf, sizeof (buf), "%s%s",
1671 				    altroot, mountpoint);
1672 			else
1673 				(void) snprintf(buf, sizeof (buf), "%s",
1674 				    mountpoint);
1675 		}
1676 
1677 		if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1678 			(void) fprintf(stderr, gettext("mountpoint '%s' : "
1679 			    "%s\n"), buf, strerror(errno));
1680 			(void) fprintf(stderr, gettext("use '-m' "
1681 			    "option to provide a different default\n"));
1682 			goto errout;
1683 		} else if (dirp) {
1684 			int count = 0;
1685 
1686 			while (count < 3 && readdir(dirp) != NULL)
1687 				count++;
1688 			(void) closedir(dirp);
1689 
1690 			if (count > 2) {
1691 				(void) fprintf(stderr, gettext("mountpoint "
1692 				    "'%s' exists and is not empty\n"), buf);
1693 				(void) fprintf(stderr, gettext("use '-m' "
1694 				    "option to provide a "
1695 				    "different default\n"));
1696 				goto errout;
1697 			}
1698 		}
1699 	}
1700 
1701 	/*
1702 	 * Now that the mountpoint's validity has been checked, ensure that
1703 	 * the property is set appropriately prior to creating the pool.
1704 	 */
1705 	if (mountpoint != NULL) {
1706 		ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1707 		    mountpoint, &fsprops, B_FALSE);
1708 		if (ret != 0)
1709 			goto errout;
1710 	}
1711 
1712 	ret = 1;
1713 	if (dryrun) {
1714 		/*
1715 		 * For a dry run invocation, print out a basic message and run
1716 		 * through all the vdevs in the list and print out in an
1717 		 * appropriate hierarchy.
1718 		 */
1719 		(void) printf(gettext("would create '%s' with the "
1720 		    "following layout:\n\n"), poolname);
1721 
1722 		print_vdev_tree(NULL, poolname, nvroot, 0, "", 0);
1723 		print_vdev_tree(NULL, "dedup", nvroot, 0,
1724 		    VDEV_ALLOC_BIAS_DEDUP, 0);
1725 		print_vdev_tree(NULL, "special", nvroot, 0,
1726 		    VDEV_ALLOC_BIAS_SPECIAL, 0);
1727 		print_vdev_tree(NULL, "logs", nvroot, 0,
1728 		    VDEV_ALLOC_BIAS_LOG, 0);
1729 		print_cache_list(nvroot, 0);
1730 		print_spare_list(nvroot, 0);
1731 
1732 		ret = 0;
1733 	} else {
1734 		/*
1735 		 * Load in feature set.
1736 		 * Note: if compatibility property not given, we'll have
1737 		 * NULL, which means 'all features'.
1738 		 */
1739 		boolean_t requested_features[SPA_FEATURES];
1740 		if (zpool_do_load_compat(compat, requested_features) !=
1741 		    ZPOOL_COMPATIBILITY_OK)
1742 			goto errout;
1743 
1744 		/*
1745 		 * props contains list of features to enable.
1746 		 * For each feature:
1747 		 *  - remove it if feature@name=disabled
1748 		 *  - leave it there if feature@name=enabled
1749 		 *  - add it if:
1750 		 *    - enable_pool_features (ie: no '-d' or '-o version')
1751 		 *    - it's supported by the kernel module
1752 		 *    - it's in the requested feature set
1753 		 *  - warn if it's enabled but not in compat
1754 		 */
1755 		for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
1756 			char propname[MAXPATHLEN];
1757 			char *propval;
1758 			zfeature_info_t *feat = &spa_feature_table[i];
1759 
1760 			(void) snprintf(propname, sizeof (propname),
1761 			    "feature@%s", feat->fi_uname);
1762 
1763 			if (!nvlist_lookup_string(props, propname, &propval)) {
1764 				if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
1765 					(void) nvlist_remove_all(props,
1766 					    propname);
1767 				if (strcmp(propval,
1768 				    ZFS_FEATURE_ENABLED) == 0 &&
1769 				    !requested_features[i])
1770 					(void) fprintf(stderr, gettext(
1771 					    "Warning: feature \"%s\" enabled "
1772 					    "but is not in specified "
1773 					    "'compatibility' feature set.\n"),
1774 					    feat->fi_uname);
1775 			} else if (
1776 			    enable_pool_features &&
1777 			    feat->fi_zfs_mod_supported &&
1778 			    requested_features[i]) {
1779 				ret = add_prop_list(propname,
1780 				    ZFS_FEATURE_ENABLED, &props, B_TRUE);
1781 				if (ret != 0)
1782 					goto errout;
1783 			}
1784 		}
1785 
1786 		ret = 1;
1787 		if (zpool_create(g_zfs, poolname,
1788 		    nvroot, props, fsprops) == 0) {
1789 			zfs_handle_t *pool = zfs_open(g_zfs,
1790 			    tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1791 			if (pool != NULL) {
1792 				if (zfs_mount(pool, NULL, 0) == 0) {
1793 					ret = zfs_shareall(pool);
1794 					zfs_commit_all_shares();
1795 				}
1796 				zfs_close(pool);
1797 			}
1798 		} else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1799 			(void) fprintf(stderr, gettext("pool name may have "
1800 			    "been omitted\n"));
1801 		}
1802 	}
1803 
1804 errout:
1805 	nvlist_free(nvroot);
1806 	nvlist_free(fsprops);
1807 	nvlist_free(props);
1808 	return (ret);
1809 badusage:
1810 	nvlist_free(fsprops);
1811 	nvlist_free(props);
1812 	usage(B_FALSE);
1813 	return (2);
1814 }
1815 
1816 /*
1817  * zpool destroy <pool>
1818  *
1819  * 	-f	Forcefully unmount any datasets
1820  *
1821  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1822  */
1823 int
1824 zpool_do_destroy(int argc, char **argv)
1825 {
1826 	boolean_t force = B_FALSE;
1827 	int c;
1828 	char *pool;
1829 	zpool_handle_t *zhp;
1830 	int ret;
1831 
1832 	/* check options */
1833 	while ((c = getopt(argc, argv, "f")) != -1) {
1834 		switch (c) {
1835 		case 'f':
1836 			force = B_TRUE;
1837 			break;
1838 		case '?':
1839 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1840 			    optopt);
1841 			usage(B_FALSE);
1842 		}
1843 	}
1844 
1845 	argc -= optind;
1846 	argv += optind;
1847 
1848 	/* check arguments */
1849 	if (argc < 1) {
1850 		(void) fprintf(stderr, gettext("missing pool argument\n"));
1851 		usage(B_FALSE);
1852 	}
1853 	if (argc > 1) {
1854 		(void) fprintf(stderr, gettext("too many arguments\n"));
1855 		usage(B_FALSE);
1856 	}
1857 
1858 	pool = argv[0];
1859 
1860 	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1861 		/*
1862 		 * As a special case, check for use of '/' in the name, and
1863 		 * direct the user to use 'zfs destroy' instead.
1864 		 */
1865 		if (strchr(pool, '/') != NULL)
1866 			(void) fprintf(stderr, gettext("use 'zfs destroy' to "
1867 			    "destroy a dataset\n"));
1868 		return (1);
1869 	}
1870 
1871 	if (zpool_disable_datasets(zhp, force) != 0) {
1872 		(void) fprintf(stderr, gettext("could not destroy '%s': "
1873 		    "could not unmount datasets\n"), zpool_get_name(zhp));
1874 		zpool_close(zhp);
1875 		return (1);
1876 	}
1877 
1878 	/* The history must be logged as part of the export */
1879 	log_history = B_FALSE;
1880 
1881 	ret = (zpool_destroy(zhp, history_str) != 0);
1882 
1883 	zpool_close(zhp);
1884 
1885 	return (ret);
1886 }
1887 
1888 typedef struct export_cbdata {
1889 	boolean_t force;
1890 	boolean_t hardforce;
1891 } export_cbdata_t;
1892 
1893 /*
1894  * Export one pool
1895  */
1896 static int
1897 zpool_export_one(zpool_handle_t *zhp, void *data)
1898 {
1899 	export_cbdata_t *cb = data;
1900 
1901 	if (zpool_disable_datasets(zhp, cb->force) != 0)
1902 		return (1);
1903 
1904 	/* The history must be logged as part of the export */
1905 	log_history = B_FALSE;
1906 
1907 	if (cb->hardforce) {
1908 		if (zpool_export_force(zhp, history_str) != 0)
1909 			return (1);
1910 	} else if (zpool_export(zhp, cb->force, history_str) != 0) {
1911 		return (1);
1912 	}
1913 
1914 	return (0);
1915 }
1916 
1917 /*
1918  * zpool export [-f] <pool> ...
1919  *
1920  *	-a	Export all pools
1921  *	-f	Forcefully unmount datasets
1922  *
1923  * Export the given pools.  By default, the command will attempt to cleanly
1924  * unmount any active datasets within the pool.  If the '-f' flag is specified,
1925  * then the datasets will be forcefully unmounted.
1926  */
1927 int
1928 zpool_do_export(int argc, char **argv)
1929 {
1930 	export_cbdata_t cb;
1931 	boolean_t do_all = B_FALSE;
1932 	boolean_t force = B_FALSE;
1933 	boolean_t hardforce = B_FALSE;
1934 	int c, ret;
1935 
1936 	/* check options */
1937 	while ((c = getopt(argc, argv, "afF")) != -1) {
1938 		switch (c) {
1939 		case 'a':
1940 			do_all = B_TRUE;
1941 			break;
1942 		case 'f':
1943 			force = B_TRUE;
1944 			break;
1945 		case 'F':
1946 			hardforce = B_TRUE;
1947 			break;
1948 		case '?':
1949 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1950 			    optopt);
1951 			usage(B_FALSE);
1952 		}
1953 	}
1954 
1955 	cb.force = force;
1956 	cb.hardforce = hardforce;
1957 	argc -= optind;
1958 	argv += optind;
1959 
1960 	if (do_all) {
1961 		if (argc != 0) {
1962 			(void) fprintf(stderr, gettext("too many arguments\n"));
1963 			usage(B_FALSE);
1964 		}
1965 
1966 		return (for_each_pool(argc, argv, B_TRUE, NULL,
1967 		    ZFS_TYPE_POOL, B_FALSE, zpool_export_one, &cb));
1968 	}
1969 
1970 	/* check arguments */
1971 	if (argc < 1) {
1972 		(void) fprintf(stderr, gettext("missing pool argument\n"));
1973 		usage(B_FALSE);
1974 	}
1975 
1976 	ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
1977 	    B_FALSE, zpool_export_one, &cb);
1978 
1979 	return (ret);
1980 }
1981 
1982 /*
1983  * Given a vdev configuration, determine the maximum width needed for the device
1984  * name column.
1985  */
1986 static int
1987 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
1988     int name_flags)
1989 {
1990 	char *name;
1991 	nvlist_t **child;
1992 	uint_t c, children;
1993 	int ret;
1994 
1995 	name = zpool_vdev_name(g_zfs, zhp, nv, name_flags);
1996 	if (strlen(name) + depth > max)
1997 		max = strlen(name) + depth;
1998 
1999 	free(name);
2000 
2001 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
2002 	    &child, &children) == 0) {
2003 		for (c = 0; c < children; c++)
2004 			if ((ret = max_width(zhp, child[c], depth + 2,
2005 			    max, name_flags)) > max)
2006 				max = ret;
2007 	}
2008 
2009 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
2010 	    &child, &children) == 0) {
2011 		for (c = 0; c < children; c++)
2012 			if ((ret = max_width(zhp, child[c], depth + 2,
2013 			    max, name_flags)) > max)
2014 				max = ret;
2015 	}
2016 
2017 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2018 	    &child, &children) == 0) {
2019 		for (c = 0; c < children; c++)
2020 			if ((ret = max_width(zhp, child[c], depth + 2,
2021 			    max, name_flags)) > max)
2022 				max = ret;
2023 	}
2024 
2025 	return (max);
2026 }
2027 
2028 typedef struct spare_cbdata {
2029 	uint64_t	cb_guid;
2030 	zpool_handle_t	*cb_zhp;
2031 } spare_cbdata_t;
2032 
2033 static boolean_t
2034 find_vdev(nvlist_t *nv, uint64_t search)
2035 {
2036 	uint64_t guid;
2037 	nvlist_t **child;
2038 	uint_t c, children;
2039 
2040 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
2041 	    search == guid)
2042 		return (B_TRUE);
2043 
2044 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2045 	    &child, &children) == 0) {
2046 		for (c = 0; c < children; c++)
2047 			if (find_vdev(child[c], search))
2048 				return (B_TRUE);
2049 	}
2050 
2051 	return (B_FALSE);
2052 }
2053 
2054 static int
2055 find_spare(zpool_handle_t *zhp, void *data)
2056 {
2057 	spare_cbdata_t *cbp = data;
2058 	nvlist_t *config, *nvroot;
2059 
2060 	config = zpool_get_config(zhp, NULL);
2061 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2062 	    &nvroot) == 0);
2063 
2064 	if (find_vdev(nvroot, cbp->cb_guid)) {
2065 		cbp->cb_zhp = zhp;
2066 		return (1);
2067 	}
2068 
2069 	zpool_close(zhp);
2070 	return (0);
2071 }
2072 
2073 typedef struct status_cbdata {
2074 	int		cb_count;
2075 	int		cb_name_flags;
2076 	int		cb_namewidth;
2077 	boolean_t	cb_allpools;
2078 	boolean_t	cb_verbose;
2079 	boolean_t	cb_literal;
2080 	boolean_t	cb_explain;
2081 	boolean_t	cb_first;
2082 	boolean_t	cb_dedup_stats;
2083 	boolean_t	cb_print_status;
2084 	boolean_t	cb_print_slow_ios;
2085 	boolean_t	cb_print_vdev_init;
2086 	boolean_t	cb_print_vdev_trim;
2087 	vdev_cmd_data_list_t	*vcdl;
2088 } status_cbdata_t;
2089 
2090 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
2091 static int
2092 is_blank_str(char *str)
2093 {
2094 	while (str != NULL && *str != '\0') {
2095 		if (!isblank(*str))
2096 			return (0);
2097 		str++;
2098 	}
2099 	return (1);
2100 }
2101 
2102 /* Print command output lines for specific vdev in a specific pool */
2103 static void
2104 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, char *path)
2105 {
2106 	vdev_cmd_data_t *data;
2107 	int i, j;
2108 	char *val;
2109 
2110 	for (i = 0; i < vcdl->count; i++) {
2111 		if ((strcmp(vcdl->data[i].path, path) != 0) ||
2112 		    (strcmp(vcdl->data[i].pool, pool) != 0)) {
2113 			/* Not the vdev we're looking for */
2114 			continue;
2115 		}
2116 
2117 		data = &vcdl->data[i];
2118 		/* Print out all the output values for this vdev */
2119 		for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
2120 			val = NULL;
2121 			/* Does this vdev have values for this column? */
2122 			for (int k = 0; k < data->cols_cnt; k++) {
2123 				if (strcmp(data->cols[k],
2124 				    vcdl->uniq_cols[j]) == 0) {
2125 					/* yes it does, record the value */
2126 					val = data->lines[k];
2127 					break;
2128 				}
2129 			}
2130 			/*
2131 			 * Mark empty values with dashes to make output
2132 			 * awk-able.
2133 			 */
2134 			if (val == NULL || is_blank_str(val))
2135 				val = "-";
2136 
2137 			printf("%*s", vcdl->uniq_cols_width[j], val);
2138 			if (j < vcdl->uniq_cols_cnt - 1)
2139 				printf("  ");
2140 		}
2141 
2142 		/* Print out any values that aren't in a column at the end */
2143 		for (j = data->cols_cnt; j < data->lines_cnt; j++) {
2144 			/* Did we have any columns?  If so print a spacer. */
2145 			if (vcdl->uniq_cols_cnt > 0)
2146 				printf("  ");
2147 
2148 			val = data->lines[j];
2149 			printf("%s", val ? val : "");
2150 		}
2151 		break;
2152 	}
2153 }
2154 
2155 /*
2156  * Print vdev initialization status for leaves
2157  */
2158 static void
2159 print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
2160 {
2161 	if (verbose) {
2162 		if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE ||
2163 		    vs->vs_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
2164 		    vs->vs_initialize_state == VDEV_INITIALIZE_COMPLETE) &&
2165 		    !vs->vs_scan_removing) {
2166 			char zbuf[1024];
2167 			char tbuf[256];
2168 			struct tm zaction_ts;
2169 
2170 			time_t t = vs->vs_initialize_action_time;
2171 			int initialize_pct = 100;
2172 			if (vs->vs_initialize_state !=
2173 			    VDEV_INITIALIZE_COMPLETE) {
2174 				initialize_pct = (vs->vs_initialize_bytes_done *
2175 				    100 / (vs->vs_initialize_bytes_est + 1));
2176 			}
2177 
2178 			(void) localtime_r(&t, &zaction_ts);
2179 			(void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
2180 
2181 			switch (vs->vs_initialize_state) {
2182 			case VDEV_INITIALIZE_SUSPENDED:
2183 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2184 				    gettext("suspended, started at"), tbuf);
2185 				break;
2186 			case VDEV_INITIALIZE_ACTIVE:
2187 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2188 				    gettext("started at"), tbuf);
2189 				break;
2190 			case VDEV_INITIALIZE_COMPLETE:
2191 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2192 				    gettext("completed at"), tbuf);
2193 				break;
2194 			}
2195 
2196 			(void) printf(gettext("  (%d%% initialized%s)"),
2197 			    initialize_pct, zbuf);
2198 		} else {
2199 			(void) printf(gettext("  (uninitialized)"));
2200 		}
2201 	} else if (vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE) {
2202 		(void) printf(gettext("  (initializing)"));
2203 	}
2204 }
2205 
2206 /*
2207  * Print vdev TRIM status for leaves
2208  */
2209 static void
2210 print_status_trim(vdev_stat_t *vs, boolean_t verbose)
2211 {
2212 	if (verbose) {
2213 		if ((vs->vs_trim_state == VDEV_TRIM_ACTIVE ||
2214 		    vs->vs_trim_state == VDEV_TRIM_SUSPENDED ||
2215 		    vs->vs_trim_state == VDEV_TRIM_COMPLETE) &&
2216 		    !vs->vs_scan_removing) {
2217 			char zbuf[1024];
2218 			char tbuf[256];
2219 			struct tm zaction_ts;
2220 
2221 			time_t t = vs->vs_trim_action_time;
2222 			int trim_pct = 100;
2223 			if (vs->vs_trim_state != VDEV_TRIM_COMPLETE) {
2224 				trim_pct = (vs->vs_trim_bytes_done *
2225 				    100 / (vs->vs_trim_bytes_est + 1));
2226 			}
2227 
2228 			(void) localtime_r(&t, &zaction_ts);
2229 			(void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
2230 
2231 			switch (vs->vs_trim_state) {
2232 			case VDEV_TRIM_SUSPENDED:
2233 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2234 				    gettext("suspended, started at"), tbuf);
2235 				break;
2236 			case VDEV_TRIM_ACTIVE:
2237 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2238 				    gettext("started at"), tbuf);
2239 				break;
2240 			case VDEV_TRIM_COMPLETE:
2241 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2242 				    gettext("completed at"), tbuf);
2243 				break;
2244 			}
2245 
2246 			(void) printf(gettext("  (%d%% trimmed%s)"),
2247 			    trim_pct, zbuf);
2248 		} else if (vs->vs_trim_notsup) {
2249 			(void) printf(gettext("  (trim unsupported)"));
2250 		} else {
2251 			(void) printf(gettext("  (untrimmed)"));
2252 		}
2253 	} else if (vs->vs_trim_state == VDEV_TRIM_ACTIVE) {
2254 		(void) printf(gettext("  (trimming)"));
2255 	}
2256 }
2257 
2258 /*
2259  * Return the color associated with a health string.  This includes returning
2260  * NULL for no color change.
2261  */
2262 static char *
2263 health_str_to_color(const char *health)
2264 {
2265 	if (strcmp(health, gettext("FAULTED")) == 0 ||
2266 	    strcmp(health, gettext("SUSPENDED")) == 0 ||
2267 	    strcmp(health, gettext("UNAVAIL")) == 0) {
2268 		return (ANSI_RED);
2269 	}
2270 
2271 	if (strcmp(health, gettext("OFFLINE")) == 0 ||
2272 	    strcmp(health, gettext("DEGRADED")) == 0 ||
2273 	    strcmp(health, gettext("REMOVED")) == 0) {
2274 		return (ANSI_YELLOW);
2275 	}
2276 
2277 	return (NULL);
2278 }
2279 
2280 /*
2281  * Print out configuration state as requested by status_callback.
2282  */
2283 static void
2284 print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
2285     nvlist_t *nv, int depth, boolean_t isspare, vdev_rebuild_stat_t *vrs)
2286 {
2287 	nvlist_t **child, *root;
2288 	uint_t c, i, vsc, children;
2289 	pool_scan_stat_t *ps = NULL;
2290 	vdev_stat_t *vs;
2291 	char rbuf[6], wbuf[6], cbuf[6];
2292 	char *vname;
2293 	uint64_t notpresent;
2294 	spare_cbdata_t spare_cb;
2295 	const char *state;
2296 	char *type;
2297 	char *path = NULL;
2298 	char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL;
2299 
2300 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2301 	    &child, &children) != 0)
2302 		children = 0;
2303 
2304 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2305 	    (uint64_t **)&vs, &vsc) == 0);
2306 
2307 	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2308 
2309 	if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2310 		return;
2311 
2312 	state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2313 
2314 	if (isspare) {
2315 		/*
2316 		 * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
2317 		 * online drives.
2318 		 */
2319 		if (vs->vs_aux == VDEV_AUX_SPARED)
2320 			state = gettext("INUSE");
2321 		else if (vs->vs_state == VDEV_STATE_HEALTHY)
2322 			state = gettext("AVAIL");
2323 	}
2324 
2325 	printf_color(health_str_to_color(state),
2326 	    "\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
2327 	    name, state);
2328 
2329 	if (!isspare) {
2330 		if (vs->vs_read_errors)
2331 			rcolor = ANSI_RED;
2332 
2333 		if (vs->vs_write_errors)
2334 			wcolor = ANSI_RED;
2335 
2336 		if (vs->vs_checksum_errors)
2337 			ccolor = ANSI_RED;
2338 
2339 		if (cb->cb_literal) {
2340 			printf(" ");
2341 			printf_color(rcolor, "%5llu",
2342 			    (u_longlong_t)vs->vs_read_errors);
2343 			printf(" ");
2344 			printf_color(wcolor, "%5llu",
2345 			    (u_longlong_t)vs->vs_write_errors);
2346 			printf(" ");
2347 			printf_color(ccolor, "%5llu",
2348 			    (u_longlong_t)vs->vs_checksum_errors);
2349 		} else {
2350 			zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
2351 			zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
2352 			zfs_nicenum(vs->vs_checksum_errors, cbuf,
2353 			    sizeof (cbuf));
2354 			printf(" ");
2355 			printf_color(rcolor, "%5s", rbuf);
2356 			printf(" ");
2357 			printf_color(wcolor, "%5s", wbuf);
2358 			printf(" ");
2359 			printf_color(ccolor, "%5s", cbuf);
2360 		}
2361 		if (cb->cb_print_slow_ios) {
2362 			if (children == 0)  {
2363 				/* Only leafs vdevs have slow IOs */
2364 				zfs_nicenum(vs->vs_slow_ios, rbuf,
2365 				    sizeof (rbuf));
2366 			} else {
2367 				snprintf(rbuf, sizeof (rbuf), "-");
2368 			}
2369 
2370 			if (cb->cb_literal)
2371 				printf(" %5llu", (u_longlong_t)vs->vs_slow_ios);
2372 			else
2373 				printf(" %5s", rbuf);
2374 		}
2375 	}
2376 
2377 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
2378 	    &notpresent) == 0) {
2379 		verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
2380 		(void) printf("  %s %s", gettext("was"), path);
2381 	} else if (vs->vs_aux != 0) {
2382 		(void) printf("  ");
2383 		color_start(ANSI_RED);
2384 		switch (vs->vs_aux) {
2385 		case VDEV_AUX_OPEN_FAILED:
2386 			(void) printf(gettext("cannot open"));
2387 			break;
2388 
2389 		case VDEV_AUX_BAD_GUID_SUM:
2390 			(void) printf(gettext("missing device"));
2391 			break;
2392 
2393 		case VDEV_AUX_NO_REPLICAS:
2394 			(void) printf(gettext("insufficient replicas"));
2395 			break;
2396 
2397 		case VDEV_AUX_VERSION_NEWER:
2398 			(void) printf(gettext("newer version"));
2399 			break;
2400 
2401 		case VDEV_AUX_UNSUP_FEAT:
2402 			(void) printf(gettext("unsupported feature(s)"));
2403 			break;
2404 
2405 		case VDEV_AUX_ASHIFT_TOO_BIG:
2406 			(void) printf(gettext("unsupported minimum blocksize"));
2407 			break;
2408 
2409 		case VDEV_AUX_SPARED:
2410 			verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
2411 			    &spare_cb.cb_guid) == 0);
2412 			if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
2413 				if (strcmp(zpool_get_name(spare_cb.cb_zhp),
2414 				    zpool_get_name(zhp)) == 0)
2415 					(void) printf(gettext("currently in "
2416 					    "use"));
2417 				else
2418 					(void) printf(gettext("in use by "
2419 					    "pool '%s'"),
2420 					    zpool_get_name(spare_cb.cb_zhp));
2421 				zpool_close(spare_cb.cb_zhp);
2422 			} else {
2423 				(void) printf(gettext("currently in use"));
2424 			}
2425 			break;
2426 
2427 		case VDEV_AUX_ERR_EXCEEDED:
2428 			(void) printf(gettext("too many errors"));
2429 			break;
2430 
2431 		case VDEV_AUX_IO_FAILURE:
2432 			(void) printf(gettext("experienced I/O failures"));
2433 			break;
2434 
2435 		case VDEV_AUX_BAD_LOG:
2436 			(void) printf(gettext("bad intent log"));
2437 			break;
2438 
2439 		case VDEV_AUX_EXTERNAL:
2440 			(void) printf(gettext("external device fault"));
2441 			break;
2442 
2443 		case VDEV_AUX_SPLIT_POOL:
2444 			(void) printf(gettext("split into new pool"));
2445 			break;
2446 
2447 		case VDEV_AUX_ACTIVE:
2448 			(void) printf(gettext("currently in use"));
2449 			break;
2450 
2451 		case VDEV_AUX_CHILDREN_OFFLINE:
2452 			(void) printf(gettext("all children offline"));
2453 			break;
2454 
2455 		case VDEV_AUX_BAD_LABEL:
2456 			(void) printf(gettext("invalid label"));
2457 			break;
2458 
2459 		default:
2460 			(void) printf(gettext("corrupted data"));
2461 			break;
2462 		}
2463 		color_end();
2464 	} else if (children == 0 && !isspare &&
2465 	    getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") == NULL &&
2466 	    VDEV_STAT_VALID(vs_physical_ashift, vsc) &&
2467 	    vs->vs_configured_ashift < vs->vs_physical_ashift) {
2468 		(void) printf(
2469 		    gettext("  block size: %dB configured, %dB native"),
2470 		    1 << vs->vs_configured_ashift, 1 << vs->vs_physical_ashift);
2471 	}
2472 
2473 	if (vs->vs_scan_removing != 0) {
2474 		(void) printf(gettext("  (removing)"));
2475 	} else if (vs->vs_noalloc != 0) {
2476 		(void) printf(gettext("  (non-allocating)"));
2477 	}
2478 
2479 	/* The root vdev has the scrub/resilver stats */
2480 	root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
2481 	    ZPOOL_CONFIG_VDEV_TREE);
2482 	(void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS,
2483 	    (uint64_t **)&ps, &c);
2484 
2485 	if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0) {
2486 		if (vs->vs_scan_processed != 0) {
2487 			(void) printf(gettext("  (%s)"),
2488 			    (ps->pss_func == POOL_SCAN_RESILVER) ?
2489 			    "resilvering" : "repairing");
2490 		} else if (vs->vs_resilver_deferred) {
2491 			(void) printf(gettext("  (awaiting resilver)"));
2492 		}
2493 	}
2494 
2495 	/* The top-level vdevs have the rebuild stats */
2496 	if (vrs != NULL && vrs->vrs_state == VDEV_REBUILD_ACTIVE &&
2497 	    children == 0) {
2498 		if (vs->vs_rebuild_processed != 0) {
2499 			(void) printf(gettext("  (resilvering)"));
2500 		}
2501 	}
2502 
2503 	if (cb->vcdl != NULL) {
2504 		if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0) {
2505 			printf("  ");
2506 			zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
2507 		}
2508 	}
2509 
2510 	/* Display vdev initialization and trim status for leaves. */
2511 	if (children == 0) {
2512 		print_status_initialize(vs, cb->cb_print_vdev_init);
2513 		print_status_trim(vs, cb->cb_print_vdev_trim);
2514 	}
2515 
2516 	(void) printf("\n");
2517 
2518 	for (c = 0; c < children; c++) {
2519 		uint64_t islog = B_FALSE, ishole = B_FALSE;
2520 
2521 		/* Don't print logs or holes here */
2522 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2523 		    &islog);
2524 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
2525 		    &ishole);
2526 		if (islog || ishole)
2527 			continue;
2528 		/* Only print normal classes here */
2529 		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2530 			continue;
2531 
2532 		/* Provide vdev_rebuild_stats to children if available */
2533 		if (vrs == NULL) {
2534 			(void) nvlist_lookup_uint64_array(nv,
2535 			    ZPOOL_CONFIG_REBUILD_STATS,
2536 			    (uint64_t **)&vrs, &i);
2537 		}
2538 
2539 		vname = zpool_vdev_name(g_zfs, zhp, child[c],
2540 		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2541 		print_status_config(zhp, cb, vname, child[c], depth + 2,
2542 		    isspare, vrs);
2543 		free(vname);
2544 	}
2545 }
2546 
2547 /*
2548  * Print the configuration of an exported pool.  Iterate over all vdevs in the
2549  * pool, printing out the name and status for each one.
2550  */
2551 static void
2552 print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
2553     int depth)
2554 {
2555 	nvlist_t **child;
2556 	uint_t c, children;
2557 	vdev_stat_t *vs;
2558 	char *type, *vname;
2559 
2560 	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2561 	if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
2562 	    strcmp(type, VDEV_TYPE_HOLE) == 0)
2563 		return;
2564 
2565 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2566 	    (uint64_t **)&vs, &c) == 0);
2567 
2568 	(void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
2569 	(void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
2570 
2571 	if (vs->vs_aux != 0) {
2572 		(void) printf("  ");
2573 
2574 		switch (vs->vs_aux) {
2575 		case VDEV_AUX_OPEN_FAILED:
2576 			(void) printf(gettext("cannot open"));
2577 			break;
2578 
2579 		case VDEV_AUX_BAD_GUID_SUM:
2580 			(void) printf(gettext("missing device"));
2581 			break;
2582 
2583 		case VDEV_AUX_NO_REPLICAS:
2584 			(void) printf(gettext("insufficient replicas"));
2585 			break;
2586 
2587 		case VDEV_AUX_VERSION_NEWER:
2588 			(void) printf(gettext("newer version"));
2589 			break;
2590 
2591 		case VDEV_AUX_UNSUP_FEAT:
2592 			(void) printf(gettext("unsupported feature(s)"));
2593 			break;
2594 
2595 		case VDEV_AUX_ERR_EXCEEDED:
2596 			(void) printf(gettext("too many errors"));
2597 			break;
2598 
2599 		case VDEV_AUX_ACTIVE:
2600 			(void) printf(gettext("currently in use"));
2601 			break;
2602 
2603 		case VDEV_AUX_CHILDREN_OFFLINE:
2604 			(void) printf(gettext("all children offline"));
2605 			break;
2606 
2607 		case VDEV_AUX_BAD_LABEL:
2608 			(void) printf(gettext("invalid label"));
2609 			break;
2610 
2611 		default:
2612 			(void) printf(gettext("corrupted data"));
2613 			break;
2614 		}
2615 	}
2616 	(void) printf("\n");
2617 
2618 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2619 	    &child, &children) != 0)
2620 		return;
2621 
2622 	for (c = 0; c < children; c++) {
2623 		uint64_t is_log = B_FALSE;
2624 
2625 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2626 		    &is_log);
2627 		if (is_log)
2628 			continue;
2629 		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2630 			continue;
2631 
2632 		vname = zpool_vdev_name(g_zfs, NULL, child[c],
2633 		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2634 		print_import_config(cb, vname, child[c], depth + 2);
2635 		free(vname);
2636 	}
2637 
2638 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
2639 	    &child, &children) == 0) {
2640 		(void) printf(gettext("\tcache\n"));
2641 		for (c = 0; c < children; c++) {
2642 			vname = zpool_vdev_name(g_zfs, NULL, child[c],
2643 			    cb->cb_name_flags);
2644 			(void) printf("\t  %s\n", vname);
2645 			free(vname);
2646 		}
2647 	}
2648 
2649 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
2650 	    &child, &children) == 0) {
2651 		(void) printf(gettext("\tspares\n"));
2652 		for (c = 0; c < children; c++) {
2653 			vname = zpool_vdev_name(g_zfs, NULL, child[c],
2654 			    cb->cb_name_flags);
2655 			(void) printf("\t  %s\n", vname);
2656 			free(vname);
2657 		}
2658 	}
2659 }
2660 
2661 /*
2662  * Print specialized class vdevs.
2663  *
2664  * These are recorded as top level vdevs in the main pool child array
2665  * but with "is_log" set to 1 or an "alloc_bias" string. We use either
2666  * print_status_config() or print_import_config() to print the top level
2667  * class vdevs then any of their children (eg mirrored slogs) are printed
2668  * recursively - which works because only the top level vdev is marked.
2669  */
2670 static void
2671 print_class_vdevs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv,
2672     const char *class)
2673 {
2674 	uint_t c, children;
2675 	nvlist_t **child;
2676 	boolean_t printed = B_FALSE;
2677 
2678 	assert(zhp != NULL || !cb->cb_verbose);
2679 
2680 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
2681 	    &children) != 0)
2682 		return;
2683 
2684 	for (c = 0; c < children; c++) {
2685 		uint64_t is_log = B_FALSE;
2686 		char *bias = NULL;
2687 		char *type = NULL;
2688 
2689 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2690 		    &is_log);
2691 
2692 		if (is_log) {
2693 			bias = VDEV_ALLOC_CLASS_LOGS;
2694 		} else {
2695 			(void) nvlist_lookup_string(child[c],
2696 			    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
2697 			(void) nvlist_lookup_string(child[c],
2698 			    ZPOOL_CONFIG_TYPE, &type);
2699 		}
2700 
2701 		if (bias == NULL || strcmp(bias, class) != 0)
2702 			continue;
2703 		if (!is_log && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2704 			continue;
2705 
2706 		if (!printed) {
2707 			(void) printf("\t%s\t\n", gettext(class));
2708 			printed = B_TRUE;
2709 		}
2710 
2711 		char *name = zpool_vdev_name(g_zfs, zhp, child[c],
2712 		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2713 		if (cb->cb_print_status)
2714 			print_status_config(zhp, cb, name, child[c], 2,
2715 			    B_FALSE, NULL);
2716 		else
2717 			print_import_config(cb, name, child[c], 2);
2718 		free(name);
2719 	}
2720 }
2721 
2722 /*
2723  * Display the status for the given pool.
2724  */
2725 static int
2726 show_import(nvlist_t *config, boolean_t report_error)
2727 {
2728 	uint64_t pool_state;
2729 	vdev_stat_t *vs;
2730 	char *name;
2731 	uint64_t guid;
2732 	uint64_t hostid = 0;
2733 	char *msgid;
2734 	char *hostname = "unknown";
2735 	nvlist_t *nvroot, *nvinfo;
2736 	zpool_status_t reason;
2737 	zpool_errata_t errata;
2738 	const char *health;
2739 	uint_t vsc;
2740 	char *comment;
2741 	status_cbdata_t cb = { 0 };
2742 
2743 	verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2744 	    &name) == 0);
2745 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2746 	    &guid) == 0);
2747 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2748 	    &pool_state) == 0);
2749 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2750 	    &nvroot) == 0);
2751 
2752 	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2753 	    (uint64_t **)&vs, &vsc) == 0);
2754 	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2755 
2756 	reason = zpool_import_status(config, &msgid, &errata);
2757 
2758 	/*
2759 	 * If we're importing using a cachefile, then we won't report any
2760 	 * errors unless we are in the scan phase of the import.
2761 	 */
2762 	if (reason != ZPOOL_STATUS_OK && !report_error)
2763 		return (reason);
2764 
2765 	(void) printf(gettext("   pool: %s\n"), name);
2766 	(void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2767 	(void) printf(gettext("  state: %s"), health);
2768 	if (pool_state == POOL_STATE_DESTROYED)
2769 		(void) printf(gettext(" (DESTROYED)"));
2770 	(void) printf("\n");
2771 
2772 	switch (reason) {
2773 	case ZPOOL_STATUS_MISSING_DEV_R:
2774 	case ZPOOL_STATUS_MISSING_DEV_NR:
2775 	case ZPOOL_STATUS_BAD_GUID_SUM:
2776 		printf_color(ANSI_BOLD, gettext("status: "));
2777 		printf_color(ANSI_YELLOW, gettext("One or more devices are "
2778 		    "missing from the system.\n"));
2779 		break;
2780 
2781 	case ZPOOL_STATUS_CORRUPT_LABEL_R:
2782 	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2783 		printf_color(ANSI_BOLD, gettext("status: "));
2784 		printf_color(ANSI_YELLOW, gettext("One or more devices contains"
2785 		    " corrupted data.\n"));
2786 		break;
2787 
2788 	case ZPOOL_STATUS_CORRUPT_DATA:
2789 		(void) printf(
2790 		    gettext(" status: The pool data is corrupted.\n"));
2791 		break;
2792 
2793 	case ZPOOL_STATUS_OFFLINE_DEV:
2794 		printf_color(ANSI_BOLD, gettext("status: "));
2795 		printf_color(ANSI_YELLOW, gettext("One or more devices "
2796 		    "are offlined.\n"));
2797 		break;
2798 
2799 	case ZPOOL_STATUS_CORRUPT_POOL:
2800 		printf_color(ANSI_BOLD, gettext("status: "));
2801 		printf_color(ANSI_YELLOW, gettext("The pool metadata is "
2802 		    "corrupted.\n"));
2803 		break;
2804 
2805 	case ZPOOL_STATUS_VERSION_OLDER:
2806 		printf_color(ANSI_BOLD, gettext("status: "));
2807 		printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2808 		    "a legacy on-disk version.\n"));
2809 		break;
2810 
2811 	case ZPOOL_STATUS_VERSION_NEWER:
2812 		printf_color(ANSI_BOLD, gettext("status: "));
2813 		printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2814 		    "an incompatible version.\n"));
2815 		break;
2816 
2817 	case ZPOOL_STATUS_FEAT_DISABLED:
2818 		printf_color(ANSI_BOLD, gettext("status: "));
2819 		printf_color(ANSI_YELLOW, gettext("Some supported "
2820 		    "features are not enabled on the pool.\n\t"
2821 		    "(Note that they may be intentionally disabled "
2822 		    "if the\n\t'compatibility' property is set.)\n"));
2823 		break;
2824 
2825 	case ZPOOL_STATUS_COMPATIBILITY_ERR:
2826 		printf_color(ANSI_BOLD, gettext("status: "));
2827 		printf_color(ANSI_YELLOW, gettext("Error reading or parsing "
2828 		    "the file(s) indicated by the 'compatibility'\n"
2829 		    "property.\n"));
2830 		break;
2831 
2832 	case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
2833 		printf_color(ANSI_BOLD, gettext("status: "));
2834 		printf_color(ANSI_YELLOW, gettext("One or more features "
2835 		    "are enabled on the pool despite not being\n"
2836 		    "requested by the 'compatibility' property.\n"));
2837 		break;
2838 
2839 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
2840 		printf_color(ANSI_BOLD, gettext("status: "));
2841 		printf_color(ANSI_YELLOW, gettext("The pool uses the following "
2842 		    "feature(s) not supported on this system:\n"));
2843 		color_start(ANSI_YELLOW);
2844 		zpool_print_unsup_feat(config);
2845 		color_end();
2846 		break;
2847 
2848 	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2849 		printf_color(ANSI_BOLD, gettext("status: "));
2850 		printf_color(ANSI_YELLOW, gettext("The pool can only be "
2851 		    "accessed in read-only mode on this system. It\n\tcannot be"
2852 		    " accessed in read-write mode because it uses the "
2853 		    "following\n\tfeature(s) not supported on this system:\n"));
2854 		color_start(ANSI_YELLOW);
2855 		zpool_print_unsup_feat(config);
2856 		color_end();
2857 		break;
2858 
2859 	case ZPOOL_STATUS_HOSTID_ACTIVE:
2860 		printf_color(ANSI_BOLD, gettext("status: "));
2861 		printf_color(ANSI_YELLOW, gettext("The pool is currently "
2862 		    "imported by another system.\n"));
2863 		break;
2864 
2865 	case ZPOOL_STATUS_HOSTID_REQUIRED:
2866 		printf_color(ANSI_BOLD, gettext("status: "));
2867 		printf_color(ANSI_YELLOW, gettext("The pool has the "
2868 		    "multihost property on.  It cannot\n\tbe safely imported "
2869 		    "when the system hostid is not set.\n"));
2870 		break;
2871 
2872 	case ZPOOL_STATUS_HOSTID_MISMATCH:
2873 		printf_color(ANSI_BOLD, gettext("status: "));
2874 		printf_color(ANSI_YELLOW, gettext("The pool was last accessed "
2875 		    "by another system.\n"));
2876 		break;
2877 
2878 	case ZPOOL_STATUS_FAULTED_DEV_R:
2879 	case ZPOOL_STATUS_FAULTED_DEV_NR:
2880 		printf_color(ANSI_BOLD, gettext("status: "));
2881 		printf_color(ANSI_YELLOW, gettext("One or more devices are "
2882 		    "faulted.\n"));
2883 		break;
2884 
2885 	case ZPOOL_STATUS_BAD_LOG:
2886 		printf_color(ANSI_BOLD, gettext("status: "));
2887 		printf_color(ANSI_YELLOW, gettext("An intent log record cannot "
2888 		    "be read.\n"));
2889 		break;
2890 
2891 	case ZPOOL_STATUS_RESILVERING:
2892 	case ZPOOL_STATUS_REBUILDING:
2893 		printf_color(ANSI_BOLD, gettext("status: "));
2894 		printf_color(ANSI_YELLOW, gettext("One or more devices were "
2895 		    "being resilvered.\n"));
2896 		break;
2897 
2898 	case ZPOOL_STATUS_ERRATA:
2899 		printf_color(ANSI_BOLD, gettext("status: "));
2900 		printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
2901 		    errata);
2902 		break;
2903 
2904 	case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
2905 		printf_color(ANSI_BOLD, gettext("status: "));
2906 		printf_color(ANSI_YELLOW, gettext("One or more devices are "
2907 		    "configured to use a non-native block size.\n"
2908 		    "\tExpect reduced performance.\n"));
2909 		break;
2910 
2911 	default:
2912 		/*
2913 		 * No other status can be seen when importing pools.
2914 		 */
2915 		assert(reason == ZPOOL_STATUS_OK);
2916 	}
2917 
2918 	/*
2919 	 * Print out an action according to the overall state of the pool.
2920 	 */
2921 	if (vs->vs_state == VDEV_STATE_HEALTHY) {
2922 		if (reason == ZPOOL_STATUS_VERSION_OLDER ||
2923 		    reason == ZPOOL_STATUS_FEAT_DISABLED) {
2924 			(void) printf(gettext(" action: The pool can be "
2925 			    "imported using its name or numeric identifier, "
2926 			    "though\n\tsome features will not be available "
2927 			    "without an explicit 'zpool upgrade'.\n"));
2928 		} else if (reason == ZPOOL_STATUS_COMPATIBILITY_ERR) {
2929 			(void) printf(gettext(" action: The pool can be "
2930 			    "imported using its name or numeric\n\tidentifier, "
2931 			    "though the file(s) indicated by its "
2932 			    "'compatibility'\n\tproperty cannot be parsed at "
2933 			    "this time.\n"));
2934 		} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
2935 			(void) printf(gettext(" action: The pool can be "
2936 			    "imported using its name or numeric "
2937 			    "identifier and\n\tthe '-f' flag.\n"));
2938 		} else if (reason == ZPOOL_STATUS_ERRATA) {
2939 			switch (errata) {
2940 			case ZPOOL_ERRATA_NONE:
2941 				break;
2942 
2943 			case ZPOOL_ERRATA_ZOL_2094_SCRUB:
2944 				(void) printf(gettext(" action: The pool can "
2945 				    "be imported using its name or numeric "
2946 				    "identifier,\n\thowever there is a compat"
2947 				    "ibility issue which should be corrected"
2948 				    "\n\tby running 'zpool scrub'\n"));
2949 				break;
2950 
2951 			case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
2952 				(void) printf(gettext(" action: The pool can"
2953 				    "not be imported with this version of ZFS "
2954 				    "due to\n\tan active asynchronous destroy. "
2955 				    "Revert to an earlier version\n\tand "
2956 				    "allow the destroy to complete before "
2957 				    "updating.\n"));
2958 				break;
2959 
2960 			case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
2961 				(void) printf(gettext(" action: Existing "
2962 				    "encrypted datasets contain an on-disk "
2963 				    "incompatibility, which\n\tneeds to be "
2964 				    "corrected. Backup these datasets to new "
2965 				    "encrypted datasets\n\tand destroy the "
2966 				    "old ones.\n"));
2967 				break;
2968 
2969 			case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
2970 				(void) printf(gettext(" action: Existing "
2971 				    "encrypted snapshots and bookmarks contain "
2972 				    "an on-disk\n\tincompatibility. This may "
2973 				    "cause on-disk corruption if they are used"
2974 				    "\n\twith 'zfs recv'. To correct the "
2975 				    "issue, enable the bookmark_v2 feature.\n\t"
2976 				    "No additional action is needed if there "
2977 				    "are no encrypted snapshots or\n\t"
2978 				    "bookmarks. If preserving the encrypted "
2979 				    "snapshots and bookmarks is\n\trequired, "
2980 				    "use a non-raw send to backup and restore "
2981 				    "them. Alternately,\n\tthey may be removed"
2982 				    " to resolve the incompatibility.\n"));
2983 				break;
2984 			default:
2985 				/*
2986 				 * All errata must contain an action message.
2987 				 */
2988 				assert(0);
2989 			}
2990 		} else {
2991 			(void) printf(gettext(" action: The pool can be "
2992 			    "imported using its name or numeric "
2993 			    "identifier.\n"));
2994 		}
2995 	} else if (vs->vs_state == VDEV_STATE_DEGRADED) {
2996 		(void) printf(gettext(" action: The pool can be imported "
2997 		    "despite missing or damaged devices.  The\n\tfault "
2998 		    "tolerance of the pool may be compromised if imported.\n"));
2999 	} else {
3000 		switch (reason) {
3001 		case ZPOOL_STATUS_VERSION_NEWER:
3002 			(void) printf(gettext(" action: The pool cannot be "
3003 			    "imported.  Access the pool on a system running "
3004 			    "newer\n\tsoftware, or recreate the pool from "
3005 			    "backup.\n"));
3006 			break;
3007 		case ZPOOL_STATUS_UNSUP_FEAT_READ:
3008 			printf_color(ANSI_BOLD, gettext("action: "));
3009 			printf_color(ANSI_YELLOW, gettext("The pool cannot be "
3010 			    "imported. Access the pool on a system that "
3011 			    "supports\n\tthe required feature(s), or recreate "
3012 			    "the pool from backup.\n"));
3013 			break;
3014 		case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
3015 			printf_color(ANSI_BOLD, gettext("action: "));
3016 			printf_color(ANSI_YELLOW, gettext("The pool cannot be "
3017 			    "imported in read-write mode. Import the pool "
3018 			    "with\n"
3019 			    "\t\"-o readonly=on\", access the pool on a system "
3020 			    "that supports the\n\trequired feature(s), or "
3021 			    "recreate the pool from backup.\n"));
3022 			break;
3023 		case ZPOOL_STATUS_MISSING_DEV_R:
3024 		case ZPOOL_STATUS_MISSING_DEV_NR:
3025 		case ZPOOL_STATUS_BAD_GUID_SUM:
3026 			(void) printf(gettext(" action: The pool cannot be "
3027 			    "imported. Attach the missing\n\tdevices and try "
3028 			    "again.\n"));
3029 			break;
3030 		case ZPOOL_STATUS_HOSTID_ACTIVE:
3031 			VERIFY0(nvlist_lookup_nvlist(config,
3032 			    ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
3033 
3034 			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
3035 				hostname = fnvlist_lookup_string(nvinfo,
3036 				    ZPOOL_CONFIG_MMP_HOSTNAME);
3037 
3038 			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
3039 				hostid = fnvlist_lookup_uint64(nvinfo,
3040 				    ZPOOL_CONFIG_MMP_HOSTID);
3041 
3042 			(void) printf(gettext(" action: The pool must be "
3043 			    "exported from %s (hostid=%lx)\n\tbefore it "
3044 			    "can be safely imported.\n"), hostname,
3045 			    (unsigned long) hostid);
3046 			break;
3047 		case ZPOOL_STATUS_HOSTID_REQUIRED:
3048 			(void) printf(gettext(" action: Set a unique system "
3049 			    "hostid with the zgenhostid(8) command.\n"));
3050 			break;
3051 		default:
3052 			(void) printf(gettext(" action: The pool cannot be "
3053 			    "imported due to damaged devices or data.\n"));
3054 		}
3055 	}
3056 
3057 	/* Print the comment attached to the pool. */
3058 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
3059 		(void) printf(gettext("comment: %s\n"), comment);
3060 
3061 	/*
3062 	 * If the state is "closed" or "can't open", and the aux state
3063 	 * is "corrupt data":
3064 	 */
3065 	if (((vs->vs_state == VDEV_STATE_CLOSED) ||
3066 	    (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
3067 	    (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
3068 		if (pool_state == POOL_STATE_DESTROYED)
3069 			(void) printf(gettext("\tThe pool was destroyed, "
3070 			    "but can be imported using the '-Df' flags.\n"));
3071 		else if (pool_state != POOL_STATE_EXPORTED)
3072 			(void) printf(gettext("\tThe pool may be active on "
3073 			    "another system, but can be imported using\n\t"
3074 			    "the '-f' flag.\n"));
3075 	}
3076 
3077 	if (msgid != NULL) {
3078 		(void) printf(gettext(
3079 		    "   see: https://openzfs.github.io/openzfs-docs/msg/%s\n"),
3080 		    msgid);
3081 	}
3082 
3083 	(void) printf(gettext(" config:\n\n"));
3084 
3085 	cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
3086 	    VDEV_NAME_TYPE_ID);
3087 	if (cb.cb_namewidth < 10)
3088 		cb.cb_namewidth = 10;
3089 
3090 	print_import_config(&cb, name, nvroot, 0);
3091 
3092 	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_DEDUP);
3093 	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
3094 	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_CLASS_LOGS);
3095 
3096 	if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
3097 		(void) printf(gettext("\n\tAdditional devices are known to "
3098 		    "be part of this pool, though their\n\texact "
3099 		    "configuration cannot be determined.\n"));
3100 	}
3101 	return (0);
3102 }
3103 
3104 static boolean_t
3105 zfs_force_import_required(nvlist_t *config)
3106 {
3107 	uint64_t state;
3108 	uint64_t hostid = 0;
3109 	nvlist_t *nvinfo;
3110 
3111 	state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
3112 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
3113 
3114 	if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
3115 		return (B_TRUE);
3116 
3117 	nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3118 	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
3119 		mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
3120 		    ZPOOL_CONFIG_MMP_STATE);
3121 
3122 		if (mmp_state != MMP_STATE_INACTIVE)
3123 			return (B_TRUE);
3124 	}
3125 
3126 	return (B_FALSE);
3127 }
3128 
3129 /*
3130  * Perform the import for the given configuration.  This passes the heavy
3131  * lifting off to zpool_import_props(), and then mounts the datasets contained
3132  * within the pool.
3133  */
3134 static int
3135 do_import(nvlist_t *config, const char *newname, const char *mntopts,
3136     nvlist_t *props, int flags)
3137 {
3138 	int ret = 0;
3139 	zpool_handle_t *zhp;
3140 	char *name;
3141 	uint64_t version;
3142 
3143 	name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
3144 	version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
3145 
3146 	if (!SPA_VERSION_IS_SUPPORTED(version)) {
3147 		(void) fprintf(stderr, gettext("cannot import '%s': pool "
3148 		    "is formatted using an unsupported ZFS version\n"), name);
3149 		return (1);
3150 	} else if (zfs_force_import_required(config) &&
3151 	    !(flags & ZFS_IMPORT_ANY_HOST)) {
3152 		mmp_state_t mmp_state = MMP_STATE_INACTIVE;
3153 		nvlist_t *nvinfo;
3154 
3155 		nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3156 		if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
3157 			mmp_state = fnvlist_lookup_uint64(nvinfo,
3158 			    ZPOOL_CONFIG_MMP_STATE);
3159 
3160 		if (mmp_state == MMP_STATE_ACTIVE) {
3161 			char *hostname = "<unknown>";
3162 			uint64_t hostid = 0;
3163 
3164 			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
3165 				hostname = fnvlist_lookup_string(nvinfo,
3166 				    ZPOOL_CONFIG_MMP_HOSTNAME);
3167 
3168 			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
3169 				hostid = fnvlist_lookup_uint64(nvinfo,
3170 				    ZPOOL_CONFIG_MMP_HOSTID);
3171 
3172 			(void) fprintf(stderr, gettext("cannot import '%s': "
3173 			    "pool is imported on %s (hostid: "
3174 			    "0x%lx)\nExport the pool on the other system, "
3175 			    "then run 'zpool import'.\n"),
3176 			    name, hostname, (unsigned long) hostid);
3177 		} else if (mmp_state == MMP_STATE_NO_HOSTID) {
3178 			(void) fprintf(stderr, gettext("Cannot import '%s': "
3179 			    "pool has the multihost property on and the\n"
3180 			    "system's hostid is not set. Set a unique hostid "
3181 			    "with the zgenhostid(8) command.\n"), name);
3182 		} else {
3183 			char *hostname = "<unknown>";
3184 			uint64_t timestamp = 0;
3185 			uint64_t hostid = 0;
3186 
3187 			if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
3188 				hostname = fnvlist_lookup_string(config,
3189 				    ZPOOL_CONFIG_HOSTNAME);
3190 
3191 			if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
3192 				timestamp = fnvlist_lookup_uint64(config,
3193 				    ZPOOL_CONFIG_TIMESTAMP);
3194 
3195 			if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
3196 				hostid = fnvlist_lookup_uint64(config,
3197 				    ZPOOL_CONFIG_HOSTID);
3198 
3199 			(void) fprintf(stderr, gettext("cannot import '%s': "
3200 			    "pool was previously in use from another system.\n"
3201 			    "Last accessed by %s (hostid=%lx) at %s"
3202 			    "The pool can be imported, use 'zpool import -f' "
3203 			    "to import the pool.\n"), name, hostname,
3204 			    (unsigned long)hostid, ctime((time_t *)&timestamp));
3205 		}
3206 
3207 		return (1);
3208 	}
3209 
3210 	if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
3211 		return (1);
3212 
3213 	if (newname != NULL)
3214 		name = (char *)newname;
3215 
3216 	if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
3217 		return (1);
3218 
3219 	/*
3220 	 * Loading keys is best effort. We don't want to return immediately
3221 	 * if it fails but we do want to give the error to the caller.
3222 	 */
3223 	if (flags & ZFS_IMPORT_LOAD_KEYS) {
3224 		ret = zfs_crypto_attempt_load_keys(g_zfs, name);
3225 		if (ret != 0)
3226 			ret = 1;
3227 	}
3228 
3229 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
3230 	    !(flags & ZFS_IMPORT_ONLY) &&
3231 	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
3232 		zpool_close(zhp);
3233 		return (1);
3234 	}
3235 
3236 	zpool_close(zhp);
3237 	return (ret);
3238 }
3239 
3240 static int
3241 import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
3242     char *orig_name, char *new_name,
3243     boolean_t do_destroyed, boolean_t pool_specified, boolean_t do_all,
3244     importargs_t *import)
3245 {
3246 	nvlist_t *config = NULL;
3247 	nvlist_t *found_config = NULL;
3248 	uint64_t pool_state;
3249 
3250 	/*
3251 	 * At this point we have a list of import candidate configs. Even if
3252 	 * we were searching by pool name or guid, we still need to
3253 	 * post-process the list to deal with pool state and possible
3254 	 * duplicate names.
3255 	 */
3256 	int err = 0;
3257 	nvpair_t *elem = NULL;
3258 	boolean_t first = B_TRUE;
3259 	while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
3260 
3261 		verify(nvpair_value_nvlist(elem, &config) == 0);
3262 
3263 		verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
3264 		    &pool_state) == 0);
3265 		if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
3266 			continue;
3267 		if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
3268 			continue;
3269 
3270 		verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
3271 		    import->policy) == 0);
3272 
3273 		if (!pool_specified) {
3274 			if (first)
3275 				first = B_FALSE;
3276 			else if (!do_all)
3277 				(void) printf("\n");
3278 
3279 			if (do_all) {
3280 				err |= do_import(config, NULL, mntopts,
3281 				    props, flags);
3282 			} else {
3283 				/*
3284 				 * If we're importing from cachefile, then
3285 				 * we don't want to report errors until we
3286 				 * are in the scan phase of the import. If
3287 				 * we get an error, then we return that error
3288 				 * to invoke the scan phase.
3289 				 */
3290 				if (import->cachefile && !import->scan)
3291 					err = show_import(config, B_FALSE);
3292 				else
3293 					(void) show_import(config, B_TRUE);
3294 			}
3295 		} else if (import->poolname != NULL) {
3296 			char *name;
3297 
3298 			/*
3299 			 * We are searching for a pool based on name.
3300 			 */
3301 			verify(nvlist_lookup_string(config,
3302 			    ZPOOL_CONFIG_POOL_NAME, &name) == 0);
3303 
3304 			if (strcmp(name, import->poolname) == 0) {
3305 				if (found_config != NULL) {
3306 					(void) fprintf(stderr, gettext(
3307 					    "cannot import '%s': more than "
3308 					    "one matching pool\n"),
3309 					    import->poolname);
3310 					(void) fprintf(stderr, gettext(
3311 					    "import by numeric ID instead\n"));
3312 					err = B_TRUE;
3313 				}
3314 				found_config = config;
3315 			}
3316 		} else {
3317 			uint64_t guid;
3318 
3319 			/*
3320 			 * Search for a pool by guid.
3321 			 */
3322 			verify(nvlist_lookup_uint64(config,
3323 			    ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
3324 
3325 			if (guid == import->guid)
3326 				found_config = config;
3327 		}
3328 	}
3329 
3330 	/*
3331 	 * If we were searching for a specific pool, verify that we found a
3332 	 * pool, and then do the import.
3333 	 */
3334 	if (pool_specified && err == 0) {
3335 		if (found_config == NULL) {
3336 			(void) fprintf(stderr, gettext("cannot import '%s': "
3337 			    "no such pool available\n"), orig_name);
3338 			err = B_TRUE;
3339 		} else {
3340 			err |= do_import(found_config, new_name,
3341 			    mntopts, props, flags);
3342 		}
3343 	}
3344 
3345 	/*
3346 	 * If we were just looking for pools, report an error if none were
3347 	 * found.
3348 	 */
3349 	if (!pool_specified && first)
3350 		(void) fprintf(stderr,
3351 		    gettext("no pools available to import\n"));
3352 	return (err);
3353 }
3354 
3355 typedef struct target_exists_args {
3356 	const char	*poolname;
3357 	uint64_t	poolguid;
3358 } target_exists_args_t;
3359 
3360 static int
3361 name_or_guid_exists(zpool_handle_t *zhp, void *data)
3362 {
3363 	target_exists_args_t *args = data;
3364 	nvlist_t *config = zpool_get_config(zhp, NULL);
3365 	int found = 0;
3366 
3367 	if (config == NULL)
3368 		return (0);
3369 
3370 	if (args->poolname != NULL) {
3371 		char *pool_name;
3372 
3373 		verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
3374 		    &pool_name) == 0);
3375 		if (strcmp(pool_name, args->poolname) == 0)
3376 			found = 1;
3377 	} else {
3378 		uint64_t pool_guid;
3379 
3380 		verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
3381 		    &pool_guid) == 0);
3382 		if (pool_guid == args->poolguid)
3383 			found = 1;
3384 	}
3385 	zpool_close(zhp);
3386 
3387 	return (found);
3388 }
3389 /*
3390  * zpool checkpoint <pool>
3391  *       checkpoint --discard <pool>
3392  *
3393  *       -d         Discard the checkpoint from a checkpointed
3394  *       --discard  pool.
3395  *
3396  *       -w         Wait for discarding a checkpoint to complete.
3397  *       --wait
3398  *
3399  * Checkpoints the specified pool, by taking a "snapshot" of its
3400  * current state. A pool can only have one checkpoint at a time.
3401  */
3402 int
3403 zpool_do_checkpoint(int argc, char **argv)
3404 {
3405 	boolean_t discard, wait;
3406 	char *pool;
3407 	zpool_handle_t *zhp;
3408 	int c, err;
3409 
3410 	struct option long_options[] = {
3411 		{"discard", no_argument, NULL, 'd'},
3412 		{"wait", no_argument, NULL, 'w'},
3413 		{0, 0, 0, 0}
3414 	};
3415 
3416 	discard = B_FALSE;
3417 	wait = B_FALSE;
3418 	while ((c = getopt_long(argc, argv, ":dw", long_options, NULL)) != -1) {
3419 		switch (c) {
3420 		case 'd':
3421 			discard = B_TRUE;
3422 			break;
3423 		case 'w':
3424 			wait = B_TRUE;
3425 			break;
3426 		case '?':
3427 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3428 			    optopt);
3429 			usage(B_FALSE);
3430 		}
3431 	}
3432 
3433 	if (wait && !discard) {
3434 		(void) fprintf(stderr, gettext("--wait only valid when "
3435 		    "--discard also specified\n"));
3436 		usage(B_FALSE);
3437 	}
3438 
3439 	argc -= optind;
3440 	argv += optind;
3441 
3442 	if (argc < 1) {
3443 		(void) fprintf(stderr, gettext("missing pool argument\n"));
3444 		usage(B_FALSE);
3445 	}
3446 
3447 	if (argc > 1) {
3448 		(void) fprintf(stderr, gettext("too many arguments\n"));
3449 		usage(B_FALSE);
3450 	}
3451 
3452 	pool = argv[0];
3453 
3454 	if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
3455 		/* As a special case, check for use of '/' in the name */
3456 		if (strchr(pool, '/') != NULL)
3457 			(void) fprintf(stderr, gettext("'zpool checkpoint' "
3458 			    "doesn't work on datasets. To save the state "
3459 			    "of a dataset from a specific point in time "
3460 			    "please use 'zfs snapshot'\n"));
3461 		return (1);
3462 	}
3463 
3464 	if (discard) {
3465 		err = (zpool_discard_checkpoint(zhp) != 0);
3466 		if (err == 0 && wait)
3467 			err = zpool_wait(zhp, ZPOOL_WAIT_CKPT_DISCARD);
3468 	} else {
3469 		err = (zpool_checkpoint(zhp) != 0);
3470 	}
3471 
3472 	zpool_close(zhp);
3473 
3474 	return (err);
3475 }
3476 
3477 #define	CHECKPOINT_OPT	1024
3478 
3479 /*
3480  * zpool import [-d dir] [-D]
3481  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3482  *              [-d dir | -c cachefile | -s] [-f] -a
3483  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3484  *              [-d dir | -c cachefile | -s] [-f] [-n] [-F] <pool | id>
3485  *              [newpool]
3486  *
3487  *	-c	Read pool information from a cachefile instead of searching
3488  *		devices. If importing from a cachefile config fails, then
3489  *		fallback to searching for devices only in the directories that
3490  *		exist in the cachefile.
3491  *
3492  *	-d	Scan in a specific directory, other than /dev/.  More than
3493  *		one directory can be specified using multiple '-d' options.
3494  *
3495  *	-D	Scan for previously destroyed pools or import all or only
3496  *		specified destroyed pools.
3497  *
3498  *	-R	Temporarily import the pool, with all mountpoints relative to
3499  *		the given root.  The pool will remain exported when the machine
3500  *		is rebooted.
3501  *
3502  *	-V	Import even in the presence of faulted vdevs.  This is an
3503  *		intentionally undocumented option for testing purposes, and
3504  *		treats the pool configuration as complete, leaving any bad
3505  *		vdevs in the FAULTED state. In other words, it does verbatim
3506  *		import.
3507  *
3508  *	-f	Force import, even if it appears that the pool is active.
3509  *
3510  *	-F	Attempt rewind if necessary.
3511  *
3512  *	-n	See if rewind would work, but don't actually rewind.
3513  *
3514  *	-N	Import the pool but don't mount datasets.
3515  *
3516  *	-T	Specify a starting txg to use for import. This option is
3517  *		intentionally undocumented option for testing purposes.
3518  *
3519  *	-a	Import all pools found.
3520  *
3521  *	-l	Load encryption keys while importing.
3522  *
3523  *	-o	Set property=value and/or temporary mount options (without '=').
3524  *
3525  *	-s	Scan using the default search path, the libblkid cache will
3526  *		not be consulted.
3527  *
3528  *	--rewind-to-checkpoint
3529  *		Import the pool and revert back to the checkpoint.
3530  *
3531  * The import command scans for pools to import, and import pools based on pool
3532  * name and GUID.  The pool can also be renamed as part of the import process.
3533  */
3534 int
3535 zpool_do_import(int argc, char **argv)
3536 {
3537 	char **searchdirs = NULL;
3538 	char *env, *envdup = NULL;
3539 	int nsearch = 0;
3540 	int c;
3541 	int err = 0;
3542 	nvlist_t *pools = NULL;
3543 	boolean_t do_all = B_FALSE;
3544 	boolean_t do_destroyed = B_FALSE;
3545 	char *mntopts = NULL;
3546 	uint64_t searchguid = 0;
3547 	char *searchname = NULL;
3548 	char *propval;
3549 	nvlist_t *policy = NULL;
3550 	nvlist_t *props = NULL;
3551 	int flags = ZFS_IMPORT_NORMAL;
3552 	uint32_t rewind_policy = ZPOOL_NO_REWIND;
3553 	boolean_t dryrun = B_FALSE;
3554 	boolean_t do_rewind = B_FALSE;
3555 	boolean_t xtreme_rewind = B_FALSE;
3556 	boolean_t do_scan = B_FALSE;
3557 	boolean_t pool_exists = B_FALSE;
3558 	boolean_t pool_specified = B_FALSE;
3559 	uint64_t txg = -1ULL;
3560 	char *cachefile = NULL;
3561 	importargs_t idata = { 0 };
3562 	char *endptr;
3563 
3564 	struct option long_options[] = {
3565 		{"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
3566 		{0, 0, 0, 0}
3567 	};
3568 
3569 	/* check options */
3570 	while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX",
3571 	    long_options, NULL)) != -1) {
3572 		switch (c) {
3573 		case 'a':
3574 			do_all = B_TRUE;
3575 			break;
3576 		case 'c':
3577 			cachefile = optarg;
3578 			break;
3579 		case 'd':
3580 			searchdirs = safe_realloc(searchdirs,
3581 			    (nsearch + 1) * sizeof (char *));
3582 			searchdirs[nsearch++] = optarg;
3583 			break;
3584 		case 'D':
3585 			do_destroyed = B_TRUE;
3586 			break;
3587 		case 'f':
3588 			flags |= ZFS_IMPORT_ANY_HOST;
3589 			break;
3590 		case 'F':
3591 			do_rewind = B_TRUE;
3592 			break;
3593 		case 'l':
3594 			flags |= ZFS_IMPORT_LOAD_KEYS;
3595 			break;
3596 		case 'm':
3597 			flags |= ZFS_IMPORT_MISSING_LOG;
3598 			break;
3599 		case 'n':
3600 			dryrun = B_TRUE;
3601 			break;
3602 		case 'N':
3603 			flags |= ZFS_IMPORT_ONLY;
3604 			break;
3605 		case 'o':
3606 			if ((propval = strchr(optarg, '=')) != NULL) {
3607 				*propval = '\0';
3608 				propval++;
3609 				if (add_prop_list(optarg, propval,
3610 				    &props, B_TRUE))
3611 					goto error;
3612 			} else {
3613 				mntopts = optarg;
3614 			}
3615 			break;
3616 		case 'R':
3617 			if (add_prop_list(zpool_prop_to_name(
3618 			    ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
3619 				goto error;
3620 			if (add_prop_list_default(zpool_prop_to_name(
3621 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
3622 				goto error;
3623 			break;
3624 		case 's':
3625 			do_scan = B_TRUE;
3626 			break;
3627 		case 't':
3628 			flags |= ZFS_IMPORT_TEMP_NAME;
3629 			if (add_prop_list_default(zpool_prop_to_name(
3630 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
3631 				goto error;
3632 			break;
3633 
3634 		case 'T':
3635 			errno = 0;
3636 			txg = strtoull(optarg, &endptr, 0);
3637 			if (errno != 0 || *endptr != '\0') {
3638 				(void) fprintf(stderr,
3639 				    gettext("invalid txg value\n"));
3640 				usage(B_FALSE);
3641 			}
3642 			rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
3643 			break;
3644 		case 'V':
3645 			flags |= ZFS_IMPORT_VERBATIM;
3646 			break;
3647 		case 'X':
3648 			xtreme_rewind = B_TRUE;
3649 			break;
3650 		case CHECKPOINT_OPT:
3651 			flags |= ZFS_IMPORT_CHECKPOINT;
3652 			break;
3653 		case ':':
3654 			(void) fprintf(stderr, gettext("missing argument for "
3655 			    "'%c' option\n"), optopt);
3656 			usage(B_FALSE);
3657 			break;
3658 		case '?':
3659 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3660 			    optopt);
3661 			usage(B_FALSE);
3662 		}
3663 	}
3664 
3665 	argc -= optind;
3666 	argv += optind;
3667 
3668 	if (cachefile && nsearch != 0) {
3669 		(void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
3670 		usage(B_FALSE);
3671 	}
3672 
3673 	if (cachefile && do_scan) {
3674 		(void) fprintf(stderr, gettext("-c is incompatible with -s\n"));
3675 		usage(B_FALSE);
3676 	}
3677 
3678 	if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
3679 		(void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
3680 		usage(B_FALSE);
3681 	}
3682 
3683 	if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
3684 		(void) fprintf(stderr, gettext("-l is only meaningful during "
3685 		    "an import\n"));
3686 		usage(B_FALSE);
3687 	}
3688 
3689 	if ((dryrun || xtreme_rewind) && !do_rewind) {
3690 		(void) fprintf(stderr,
3691 		    gettext("-n or -X only meaningful with -F\n"));
3692 		usage(B_FALSE);
3693 	}
3694 	if (dryrun)
3695 		rewind_policy = ZPOOL_TRY_REWIND;
3696 	else if (do_rewind)
3697 		rewind_policy = ZPOOL_DO_REWIND;
3698 	if (xtreme_rewind)
3699 		rewind_policy |= ZPOOL_EXTREME_REWIND;
3700 
3701 	/* In the future, we can capture further policy and include it here */
3702 	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
3703 	    nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
3704 	    nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
3705 	    rewind_policy) != 0)
3706 		goto error;
3707 
3708 	/* check argument count */
3709 	if (do_all) {
3710 		if (argc != 0) {
3711 			(void) fprintf(stderr, gettext("too many arguments\n"));
3712 			usage(B_FALSE);
3713 		}
3714 	} else {
3715 		if (argc > 2) {
3716 			(void) fprintf(stderr, gettext("too many arguments\n"));
3717 			usage(B_FALSE);
3718 		}
3719 	}
3720 
3721 	/*
3722 	 * Check for the effective uid.  We do this explicitly here because
3723 	 * otherwise any attempt to discover pools will silently fail.
3724 	 */
3725 	if (argc == 0 && geteuid() != 0) {
3726 		(void) fprintf(stderr, gettext("cannot "
3727 		    "discover pools: permission denied\n"));
3728 		if (searchdirs != NULL)
3729 			free(searchdirs);
3730 
3731 		nvlist_free(props);
3732 		nvlist_free(policy);
3733 		return (1);
3734 	}
3735 
3736 	/*
3737 	 * Depending on the arguments given, we do one of the following:
3738 	 *
3739 	 *	<none>	Iterate through all pools and display information about
3740 	 *		each one.
3741 	 *
3742 	 *	-a	Iterate through all pools and try to import each one.
3743 	 *
3744 	 *	<id>	Find the pool that corresponds to the given GUID/pool
3745 	 *		name and import that one.
3746 	 *
3747 	 *	-D	Above options applies only to destroyed pools.
3748 	 */
3749 	if (argc != 0) {
3750 		char *endptr;
3751 
3752 		errno = 0;
3753 		searchguid = strtoull(argv[0], &endptr, 10);
3754 		if (errno != 0 || *endptr != '\0') {
3755 			searchname = argv[0];
3756 			searchguid = 0;
3757 		}
3758 		pool_specified = B_TRUE;
3759 
3760 		/*
3761 		 * User specified a name or guid.  Ensure it's unique.
3762 		 */
3763 		target_exists_args_t search = {searchname, searchguid};
3764 		pool_exists = zpool_iter(g_zfs, name_or_guid_exists, &search);
3765 	}
3766 
3767 	/*
3768 	 * Check the environment for the preferred search path.
3769 	 */
3770 	if ((searchdirs == NULL) && (env = getenv("ZPOOL_IMPORT_PATH"))) {
3771 		char *dir, *tmp = NULL;
3772 
3773 		envdup = strdup(env);
3774 
3775 		for (dir = strtok_r(envdup, ":", &tmp);
3776 		    dir != NULL;
3777 		    dir = strtok_r(NULL, ":", &tmp)) {
3778 			searchdirs = safe_realloc(searchdirs,
3779 			    (nsearch + 1) * sizeof (char *));
3780 			searchdirs[nsearch++] = dir;
3781 		}
3782 	}
3783 
3784 	idata.path = searchdirs;
3785 	idata.paths = nsearch;
3786 	idata.poolname = searchname;
3787 	idata.guid = searchguid;
3788 	idata.cachefile = cachefile;
3789 	idata.scan = do_scan;
3790 	idata.policy = policy;
3791 
3792 	pools = zpool_search_import(g_zfs, &idata, &libzfs_config_ops);
3793 
3794 	if (pools != NULL && pool_exists &&
3795 	    (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
3796 		(void) fprintf(stderr, gettext("cannot import '%s': "
3797 		    "a pool with that name already exists\n"),
3798 		    argv[0]);
3799 		(void) fprintf(stderr, gettext("use the form '%s "
3800 		    "<pool | id> <newpool>' to give it a new name\n"),
3801 		    "zpool import");
3802 		err = 1;
3803 	} else if (pools == NULL && pool_exists) {
3804 		(void) fprintf(stderr, gettext("cannot import '%s': "
3805 		    "a pool with that name is already created/imported,\n"),
3806 		    argv[0]);
3807 		(void) fprintf(stderr, gettext("and no additional pools "
3808 		    "with that name were found\n"));
3809 		err = 1;
3810 	} else if (pools == NULL) {
3811 		if (argc != 0) {
3812 			(void) fprintf(stderr, gettext("cannot import '%s': "
3813 			    "no such pool available\n"), argv[0]);
3814 		}
3815 		err = 1;
3816 	}
3817 
3818 	if (err == 1) {
3819 		free(searchdirs);
3820 		free(envdup);
3821 		nvlist_free(policy);
3822 		nvlist_free(pools);
3823 		nvlist_free(props);
3824 		return (1);
3825 	}
3826 
3827 	err = import_pools(pools, props, mntopts, flags,
3828 	    argc >= 1 ? argv[0] : NULL,
3829 	    argc >= 2 ? argv[1] : NULL,
3830 	    do_destroyed, pool_specified, do_all, &idata);
3831 
3832 	/*
3833 	 * If we're using the cachefile and we failed to import, then
3834 	 * fallback to scanning the directory for pools that match
3835 	 * those in the cachefile.
3836 	 */
3837 	if (err != 0 && cachefile != NULL) {
3838 		(void) printf(gettext("cachefile import failed, retrying\n"));
3839 
3840 		/*
3841 		 * We use the scan flag to gather the directories that exist
3842 		 * in the cachefile. If we need to fallback to searching for
3843 		 * the pool config, we will only search devices in these
3844 		 * directories.
3845 		 */
3846 		idata.scan = B_TRUE;
3847 		nvlist_free(pools);
3848 		pools = zpool_search_import(g_zfs, &idata, &libzfs_config_ops);
3849 
3850 		err = import_pools(pools, props, mntopts, flags,
3851 		    argc >= 1 ? argv[0] : NULL,
3852 		    argc >= 2 ? argv[1] : NULL,
3853 		    do_destroyed, pool_specified, do_all, &idata);
3854 	}
3855 
3856 error:
3857 	nvlist_free(props);
3858 	nvlist_free(pools);
3859 	nvlist_free(policy);
3860 	free(searchdirs);
3861 	free(envdup);
3862 
3863 	return (err ? 1 : 0);
3864 }
3865 
3866 /*
3867  * zpool sync [-f] [pool] ...
3868  *
3869  * -f (undocumented) force uberblock (and config including zpool cache file)
3870  *    update.
3871  *
3872  * Sync the specified pool(s).
3873  * Without arguments "zpool sync" will sync all pools.
3874  * This command initiates TXG sync(s) and will return after the TXG(s) commit.
3875  *
3876  */
3877 static int
3878 zpool_do_sync(int argc, char **argv)
3879 {
3880 	int ret;
3881 	boolean_t force = B_FALSE;
3882 
3883 	/* check options */
3884 	while ((ret  = getopt(argc, argv, "f")) != -1) {
3885 		switch (ret) {
3886 		case 'f':
3887 			force = B_TRUE;
3888 			break;
3889 		case '?':
3890 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3891 			    optopt);
3892 			usage(B_FALSE);
3893 		}
3894 	}
3895 
3896 	argc -= optind;
3897 	argv += optind;
3898 
3899 	/* if argc == 0 we will execute zpool_sync_one on all pools */
3900 	ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
3901 	    B_FALSE, zpool_sync_one, &force);
3902 
3903 	return (ret);
3904 }
3905 
3906 typedef struct iostat_cbdata {
3907 	uint64_t cb_flags;
3908 	int cb_namewidth;
3909 	int cb_iteration;
3910 	boolean_t cb_verbose;
3911 	boolean_t cb_literal;
3912 	boolean_t cb_scripted;
3913 	zpool_list_t *cb_list;
3914 	vdev_cmd_data_list_t *vcdl;
3915 	vdev_cbdata_t cb_vdevs;
3916 } iostat_cbdata_t;
3917 
3918 /*  iostat labels */
3919 typedef struct name_and_columns {
3920 	const char *name;	/* Column name */
3921 	unsigned int columns;	/* Center name to this number of columns */
3922 } name_and_columns_t;
3923 
3924 #define	IOSTAT_MAX_LABELS	15	/* Max number of labels on one line */
3925 
3926 static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
3927 {
3928 	[IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
3929 	    {NULL}},
3930 	[IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3931 	    {"asyncq_wait", 2}, {"scrub", 1}, {"trim", 1}, {"rebuild", 1},
3932 	    {NULL}},
3933 	[IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
3934 	    {"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
3935 	    {"trimq_write", 2}, {"rebuildq_write", 2}, {NULL}},
3936 	[IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3937 	    {"asyncq_wait", 2}, {NULL}},
3938 	[IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
3939 	    {"async_read", 2}, {"async_write", 2}, {"scrub", 2},
3940 	    {"trim", 2}, {"rebuild", 2}, {NULL}},
3941 };
3942 
3943 /* Shorthand - if "columns" field not set, default to 1 column */
3944 static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
3945 {
3946 	[IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
3947 	    {"write"}, {NULL}},
3948 	[IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3949 	    {"write"}, {"read"}, {"write"}, {"wait"}, {"wait"}, {"wait"},
3950 	    {NULL}},
3951 	[IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
3952 	    {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"},
3953 	    {"pend"}, {"activ"}, {"pend"}, {"activ"}, {NULL}},
3954 	[IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3955 	    {"write"}, {"read"}, {"write"}, {"scrub"}, {"trim"}, {"rebuild"},
3956 	    {NULL}},
3957 	[IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
3958 	    {"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
3959 	    {"ind"}, {"agg"}, {NULL}},
3960 };
3961 
3962 static const char *histo_to_title[] = {
3963 	[IOS_L_HISTO] = "latency",
3964 	[IOS_RQ_HISTO] = "req_size",
3965 };
3966 
3967 /*
3968  * Return the number of labels in a null-terminated name_and_columns_t
3969  * array.
3970  *
3971  */
3972 static unsigned int
3973 label_array_len(const name_and_columns_t *labels)
3974 {
3975 	int i = 0;
3976 
3977 	while (labels[i].name)
3978 		i++;
3979 
3980 	return (i);
3981 }
3982 
3983 /*
3984  * Return the number of strings in a null-terminated string array.
3985  * For example:
3986  *
3987  *     const char foo[] = {"bar", "baz", NULL}
3988  *
3989  * returns 2
3990  */
3991 static uint64_t
3992 str_array_len(const char *array[])
3993 {
3994 	uint64_t i = 0;
3995 	while (array[i])
3996 		i++;
3997 
3998 	return (i);
3999 }
4000 
4001 
4002 /*
4003  * Return a default column width for default/latency/queue columns. This does
4004  * not include histograms, which have their columns autosized.
4005  */
4006 static unsigned int
4007 default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
4008 {
4009 	unsigned long column_width = 5; /* Normal niceprint */
4010 	static unsigned long widths[] = {
4011 		/*
4012 		 * Choose some sane default column sizes for printing the
4013 		 * raw numbers.
4014 		 */
4015 		[IOS_DEFAULT] = 15, /* 1PB capacity */
4016 		[IOS_LATENCY] = 10, /* 1B ns = 10sec */
4017 		[IOS_QUEUES] = 6,   /* 1M queue entries */
4018 		[IOS_L_HISTO] = 10, /* 1B ns = 10sec */
4019 		[IOS_RQ_HISTO] = 6, /* 1M queue entries */
4020 	};
4021 
4022 	if (cb->cb_literal)
4023 		column_width = widths[type];
4024 
4025 	return (column_width);
4026 }
4027 
4028 /*
4029  * Print the column labels, i.e:
4030  *
4031  *   capacity     operations     bandwidth
4032  * alloc   free   read  write   read  write  ...
4033  *
4034  * If force_column_width is set, use it for the column width.  If not set, use
4035  * the default column width.
4036  */
4037 static void
4038 print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
4039     const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
4040 {
4041 	int i, idx, s;
4042 	int text_start, rw_column_width, spaces_to_end;
4043 	uint64_t flags = cb->cb_flags;
4044 	uint64_t f;
4045 	unsigned int column_width = force_column_width;
4046 
4047 	/* For each bit set in flags */
4048 	for (f = flags; f; f &= ~(1ULL << idx)) {
4049 		idx = lowbit64(f) - 1;
4050 		if (!force_column_width)
4051 			column_width = default_column_width(cb, idx);
4052 		/* Print our top labels centered over "read  write" label. */
4053 		for (i = 0; i < label_array_len(labels[idx]); i++) {
4054 			const char *name = labels[idx][i].name;
4055 			/*
4056 			 * We treat labels[][].columns == 0 as shorthand
4057 			 * for one column.  It makes writing out the label
4058 			 * tables more concise.
4059 			 */
4060 			unsigned int columns = MAX(1, labels[idx][i].columns);
4061 			unsigned int slen = strlen(name);
4062 
4063 			rw_column_width = (column_width * columns) +
4064 			    (2 * (columns - 1));
4065 
4066 			text_start = (int)((rw_column_width) / columns -
4067 			    slen / columns);
4068 			if (text_start < 0)
4069 				text_start = 0;
4070 
4071 			printf("  ");	/* Two spaces between columns */
4072 
4073 			/* Space from beginning of column to label */
4074 			for (s = 0; s < text_start; s++)
4075 				printf(" ");
4076 
4077 			printf("%s", name);
4078 
4079 			/* Print space after label to end of column */
4080 			spaces_to_end = rw_column_width - text_start - slen;
4081 			if (spaces_to_end < 0)
4082 				spaces_to_end = 0;
4083 
4084 			for (s = 0; s < spaces_to_end; s++)
4085 				printf(" ");
4086 		}
4087 	}
4088 }
4089 
4090 
4091 /*
4092  * print_cmd_columns - Print custom column titles from -c
4093  *
4094  * If the user specified the "zpool status|iostat -c" then print their custom
4095  * column titles in the header.  For example, print_cmd_columns() would print
4096  * the "  col1  col2" part of this:
4097  *
4098  * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
4099  * ...
4100  *	      capacity     operations     bandwidth
4101  * pool        alloc   free   read  write   read  write  col1  col2
4102  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
4103  * mypool       269K  1008M      0      0    107    946
4104  *   mirror     269K  1008M      0      0    107    946
4105  *     sdb         -      -      0      0    102    473  val1  val2
4106  *     sdc         -      -      0      0      5    473  val1  val2
4107  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
4108  */
4109 static void
4110 print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
4111 {
4112 	int i, j;
4113 	vdev_cmd_data_t *data = &vcdl->data[0];
4114 
4115 	if (vcdl->count == 0 || data == NULL)
4116 		return;
4117 
4118 	/*
4119 	 * Each vdev cmd should have the same column names unless the user did
4120 	 * something weird with their cmd.  Just take the column names from the
4121 	 * first vdev and assume it works for all of them.
4122 	 */
4123 	for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
4124 		printf("  ");
4125 		if (use_dashes) {
4126 			for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
4127 				printf("-");
4128 		} else {
4129 			printf_color(ANSI_BOLD, "%*s", vcdl->uniq_cols_width[i],
4130 			    vcdl->uniq_cols[i]);
4131 		}
4132 	}
4133 }
4134 
4135 
4136 /*
4137  * Utility function to print out a line of dashes like:
4138  *
4139  * 	--------------------------------  -----  -----  -----  -----  -----
4140  *
4141  * ...or a dashed named-row line like:
4142  *
4143  * 	logs                                  -      -      -      -      -
4144  *
4145  * @cb:				iostat data
4146  *
4147  * @force_column_width		If non-zero, use the value as the column width.
4148  * 				Otherwise use the default column widths.
4149  *
4150  * @name:			Print a dashed named-row line starting
4151  * 				with @name.  Otherwise, print a regular
4152  * 				dashed line.
4153  */
4154 static void
4155 print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
4156     const char *name)
4157 {
4158 	int i;
4159 	unsigned int namewidth;
4160 	uint64_t flags = cb->cb_flags;
4161 	uint64_t f;
4162 	int idx;
4163 	const name_and_columns_t *labels;
4164 	const char *title;
4165 
4166 
4167 	if (cb->cb_flags & IOS_ANYHISTO_M) {
4168 		title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
4169 	} else if (cb->cb_vdevs.cb_names_count) {
4170 		title = "vdev";
4171 	} else  {
4172 		title = "pool";
4173 	}
4174 
4175 	namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
4176 	    name ? strlen(name) : 0);
4177 
4178 
4179 	if (name) {
4180 		printf("%-*s", namewidth, name);
4181 	} else {
4182 		for (i = 0; i < namewidth; i++)
4183 			(void) printf("-");
4184 	}
4185 
4186 	/* For each bit in flags */
4187 	for (f = flags; f; f &= ~(1ULL << idx)) {
4188 		unsigned int column_width;
4189 		idx = lowbit64(f) - 1;
4190 		if (force_column_width)
4191 			column_width = force_column_width;
4192 		else
4193 			column_width = default_column_width(cb, idx);
4194 
4195 		labels = iostat_bottom_labels[idx];
4196 		for (i = 0; i < label_array_len(labels); i++) {
4197 			if (name)
4198 				printf("  %*s-", column_width - 1, " ");
4199 			else
4200 				printf("  %.*s", column_width,
4201 				    "--------------------");
4202 		}
4203 	}
4204 }
4205 
4206 
4207 static void
4208 print_iostat_separator_impl(iostat_cbdata_t *cb,
4209     unsigned int force_column_width)
4210 {
4211 	print_iostat_dashes(cb, force_column_width, NULL);
4212 }
4213 
4214 static void
4215 print_iostat_separator(iostat_cbdata_t *cb)
4216 {
4217 	print_iostat_separator_impl(cb, 0);
4218 }
4219 
4220 static void
4221 print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
4222     const char *histo_vdev_name)
4223 {
4224 	unsigned int namewidth;
4225 	const char *title;
4226 
4227 	if (cb->cb_flags & IOS_ANYHISTO_M) {
4228 		title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
4229 	} else if (cb->cb_vdevs.cb_names_count) {
4230 		title = "vdev";
4231 	} else  {
4232 		title = "pool";
4233 	}
4234 
4235 	namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
4236 	    histo_vdev_name ? strlen(histo_vdev_name) : 0);
4237 
4238 	if (histo_vdev_name)
4239 		printf("%-*s", namewidth, histo_vdev_name);
4240 	else
4241 		printf("%*s", namewidth, "");
4242 
4243 
4244 	print_iostat_labels(cb, force_column_width, iostat_top_labels);
4245 	printf("\n");
4246 
4247 	printf("%-*s", namewidth, title);
4248 
4249 	print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
4250 	if (cb->vcdl != NULL)
4251 		print_cmd_columns(cb->vcdl, 0);
4252 
4253 	printf("\n");
4254 
4255 	print_iostat_separator_impl(cb, force_column_width);
4256 
4257 	if (cb->vcdl != NULL)
4258 		print_cmd_columns(cb->vcdl, 1);
4259 
4260 	printf("\n");
4261 }
4262 
4263 static void
4264 print_iostat_header(iostat_cbdata_t *cb)
4265 {
4266 	print_iostat_header_impl(cb, 0, NULL);
4267 }
4268 
4269 
4270 /*
4271  * Display a single statistic.
4272  */
4273 static void
4274 print_one_stat(uint64_t value, enum zfs_nicenum_format format,
4275     unsigned int column_size, boolean_t scripted)
4276 {
4277 	char buf[64];
4278 
4279 	zfs_nicenum_format(value, buf, sizeof (buf), format);
4280 
4281 	if (scripted)
4282 		printf("\t%s", buf);
4283 	else
4284 		printf("  %*s", column_size, buf);
4285 }
4286 
4287 /*
4288  * Calculate the default vdev stats
4289  *
4290  * Subtract oldvs from newvs, apply a scaling factor, and save the resulting
4291  * stats into calcvs.
4292  */
4293 static void
4294 calc_default_iostats(vdev_stat_t *oldvs, vdev_stat_t *newvs,
4295     vdev_stat_t *calcvs)
4296 {
4297 	int i;
4298 
4299 	memcpy(calcvs, newvs, sizeof (*calcvs));
4300 	for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
4301 		calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);
4302 
4303 	for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
4304 		calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
4305 }
4306 
4307 /*
4308  * Internal representation of the extended iostats data.
4309  *
4310  * The extended iostat stats are exported in nvlists as either uint64_t arrays
4311  * or single uint64_t's.  We make both look like arrays to make them easier
4312  * to process.  In order to make single uint64_t's look like arrays, we set
4313  * __data to the stat data, and then set *data = &__data with count = 1.  Then,
4314  * we can just use *data and count.
4315  */
4316 struct stat_array {
4317 	uint64_t *data;
4318 	uint_t count;	/* Number of entries in data[] */
4319 	uint64_t __data; /* Only used when data is a single uint64_t */
4320 };
4321 
4322 static uint64_t
4323 stat_histo_max(struct stat_array *nva, unsigned int len)
4324 {
4325 	uint64_t max = 0;
4326 	int i;
4327 	for (i = 0; i < len; i++)
4328 		max = MAX(max, array64_max(nva[i].data, nva[i].count));
4329 
4330 	return (max);
4331 }
4332 
4333 /*
4334  * Helper function to lookup a uint64_t array or uint64_t value and store its
4335  * data as a stat_array.  If the nvpair is a single uint64_t value, then we make
4336  * it look like a one element array to make it easier to process.
4337  */
4338 static int
4339 nvpair64_to_stat_array(nvlist_t *nvl, const char *name,
4340     struct stat_array *nva)
4341 {
4342 	nvpair_t *tmp;
4343 	int ret;
4344 
4345 	verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
4346 	switch (nvpair_type(tmp)) {
4347 	case DATA_TYPE_UINT64_ARRAY:
4348 		ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
4349 		break;
4350 	case DATA_TYPE_UINT64:
4351 		ret = nvpair_value_uint64(tmp, &nva->__data);
4352 		nva->data = &nva->__data;
4353 		nva->count = 1;
4354 		break;
4355 	default:
4356 		/* Not a uint64_t */
4357 		ret = EINVAL;
4358 		break;
4359 	}
4360 
4361 	return (ret);
4362 }
4363 
4364 /*
4365  * Given a list of nvlist names, look up the extended stats in newnv and oldnv,
4366  * subtract them, and return the results in a newly allocated stat_array.
4367  * You must free the returned array after you are done with it with
4368  * free_calc_stats().
4369  *
4370  * Additionally, you can set "oldnv" to NULL if you simply want the newnv
4371  * values.
4372  */
4373 static struct stat_array *
4374 calc_and_alloc_stats_ex(const char **names, unsigned int len, nvlist_t *oldnv,
4375     nvlist_t *newnv)
4376 {
4377 	nvlist_t *oldnvx = NULL, *newnvx;
4378 	struct stat_array *oldnva, *newnva, *calcnva;
4379 	int i, j;
4380 	unsigned int alloc_size = (sizeof (struct stat_array)) * len;
4381 
4382 	/* Extract our extended stats nvlist from the main list */
4383 	verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4384 	    &newnvx) == 0);
4385 	if (oldnv) {
4386 		verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4387 		    &oldnvx) == 0);
4388 	}
4389 
4390 	newnva = safe_malloc(alloc_size);
4391 	oldnva = safe_malloc(alloc_size);
4392 	calcnva = safe_malloc(alloc_size);
4393 
4394 	for (j = 0; j < len; j++) {
4395 		verify(nvpair64_to_stat_array(newnvx, names[j],
4396 		    &newnva[j]) == 0);
4397 		calcnva[j].count = newnva[j].count;
4398 		alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
4399 		calcnva[j].data = safe_malloc(alloc_size);
4400 		memcpy(calcnva[j].data, newnva[j].data, alloc_size);
4401 
4402 		if (oldnvx) {
4403 			verify(nvpair64_to_stat_array(oldnvx, names[j],
4404 			    &oldnva[j]) == 0);
4405 			for (i = 0; i < oldnva[j].count; i++)
4406 				calcnva[j].data[i] -= oldnva[j].data[i];
4407 		}
4408 	}
4409 	free(newnva);
4410 	free(oldnva);
4411 	return (calcnva);
4412 }
4413 
4414 static void
4415 free_calc_stats(struct stat_array *nva, unsigned int len)
4416 {
4417 	int i;
4418 	for (i = 0; i < len; i++)
4419 		free(nva[i].data);
4420 
4421 	free(nva);
4422 }
4423 
4424 static void
4425 print_iostat_histo(struct stat_array *nva, unsigned int len,
4426     iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
4427     double scale)
4428 {
4429 	int i, j;
4430 	char buf[6];
4431 	uint64_t val;
4432 	enum zfs_nicenum_format format;
4433 	unsigned int buckets;
4434 	unsigned int start_bucket;
4435 
4436 	if (cb->cb_literal)
4437 		format = ZFS_NICENUM_RAW;
4438 	else
4439 		format = ZFS_NICENUM_1024;
4440 
4441 	/* All these histos are the same size, so just use nva[0].count */
4442 	buckets = nva[0].count;
4443 
4444 	if (cb->cb_flags & IOS_RQ_HISTO_M) {
4445 		/* Start at 512 - req size should never be lower than this */
4446 		start_bucket = 9;
4447 	} else {
4448 		start_bucket = 0;
4449 	}
4450 
4451 	for (j = start_bucket; j < buckets; j++) {
4452 		/* Print histogram bucket label */
4453 		if (cb->cb_flags & IOS_L_HISTO_M) {
4454 			/* Ending range of this bucket */
4455 			val = (1UL << (j + 1)) - 1;
4456 			zfs_nicetime(val, buf, sizeof (buf));
4457 		} else {
4458 			/* Request size (starting range of bucket) */
4459 			val = (1UL << j);
4460 			zfs_nicenum(val, buf, sizeof (buf));
4461 		}
4462 
4463 		if (cb->cb_scripted)
4464 			printf("%llu", (u_longlong_t)val);
4465 		else
4466 			printf("%-*s", namewidth, buf);
4467 
4468 		/* Print the values on the line */
4469 		for (i = 0; i < len; i++) {
4470 			print_one_stat(nva[i].data[j] * scale, format,
4471 			    column_width, cb->cb_scripted);
4472 		}
4473 		printf("\n");
4474 	}
4475 }
4476 
4477 static void
4478 print_solid_separator(unsigned int length)
4479 {
4480 	while (length--)
4481 		printf("-");
4482 	printf("\n");
4483 }
4484 
4485 static void
4486 print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv,
4487     nvlist_t *newnv, double scale, const char *name)
4488 {
4489 	unsigned int column_width;
4490 	unsigned int namewidth;
4491 	unsigned int entire_width;
4492 	enum iostat_type type;
4493 	struct stat_array *nva;
4494 	const char **names;
4495 	unsigned int names_len;
4496 
4497 	/* What type of histo are we? */
4498 	type = IOS_HISTO_IDX(cb->cb_flags);
4499 
4500 	/* Get NULL-terminated array of nvlist names for our histo */
4501 	names = vsx_type_to_nvlist[type];
4502 	names_len = str_array_len(names); /* num of names */
4503 
4504 	nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);
4505 
4506 	if (cb->cb_literal) {
4507 		column_width = MAX(5,
4508 		    (unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
4509 	} else {
4510 		column_width = 5;
4511 	}
4512 
4513 	namewidth = MAX(cb->cb_namewidth,
4514 	    strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));
4515 
4516 	/*
4517 	 * Calculate the entire line width of what we're printing.  The
4518 	 * +2 is for the two spaces between columns:
4519 	 */
4520 	/*	 read  write				*/
4521 	/*	-----  -----				*/
4522 	/*	|___|  <---------- column_width		*/
4523 	/*						*/
4524 	/*	|__________|  <--- entire_width		*/
4525 	/*						*/
4526 	entire_width = namewidth + (column_width + 2) *
4527 	    label_array_len(iostat_bottom_labels[type]);
4528 
4529 	if (cb->cb_scripted)
4530 		printf("%s\n", name);
4531 	else
4532 		print_iostat_header_impl(cb, column_width, name);
4533 
4534 	print_iostat_histo(nva, names_len, cb, column_width,
4535 	    namewidth, scale);
4536 
4537 	free_calc_stats(nva, names_len);
4538 	if (!cb->cb_scripted)
4539 		print_solid_separator(entire_width);
4540 }
4541 
4542 /*
4543  * Calculate the average latency of a power-of-two latency histogram
4544  */
4545 static uint64_t
4546 single_histo_average(uint64_t *histo, unsigned int buckets)
4547 {
4548 	int i;
4549 	uint64_t count = 0, total = 0;
4550 
4551 	for (i = 0; i < buckets; i++) {
4552 		/*
4553 		 * Our buckets are power-of-two latency ranges.  Use the
4554 		 * midpoint latency of each bucket to calculate the average.
4555 		 * For example:
4556 		 *
4557 		 * Bucket          Midpoint
4558 		 * 8ns-15ns:       12ns
4559 		 * 16ns-31ns:      24ns
4560 		 * ...
4561 		 */
4562 		if (histo[i] != 0) {
4563 			total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
4564 			count += histo[i];
4565 		}
4566 	}
4567 
4568 	/* Prevent divide by zero */
4569 	return (count == 0 ? 0 : total / count);
4570 }
4571 
4572 static void
4573 print_iostat_queues(iostat_cbdata_t *cb, nvlist_t *oldnv,
4574     nvlist_t *newnv)
4575 {
4576 	int i;
4577 	uint64_t val;
4578 	const char *names[] = {
4579 		ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
4580 		ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
4581 		ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
4582 		ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
4583 		ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
4584 		ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
4585 		ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
4586 		ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
4587 		ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
4588 		ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
4589 		ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
4590 		ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
4591 		ZPOOL_CONFIG_VDEV_REBUILD_PEND_QUEUE,
4592 		ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
4593 	};
4594 
4595 	struct stat_array *nva;
4596 
4597 	unsigned int column_width = default_column_width(cb, IOS_QUEUES);
4598 	enum zfs_nicenum_format format;
4599 
4600 	nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);
4601 
4602 	if (cb->cb_literal)
4603 		format = ZFS_NICENUM_RAW;
4604 	else
4605 		format = ZFS_NICENUM_1024;
4606 
4607 	for (i = 0; i < ARRAY_SIZE(names); i++) {
4608 		val = nva[i].data[0];
4609 		print_one_stat(val, format, column_width, cb->cb_scripted);
4610 	}
4611 
4612 	free_calc_stats(nva, ARRAY_SIZE(names));
4613 }
4614 
4615 static void
4616 print_iostat_latency(iostat_cbdata_t *cb, nvlist_t *oldnv,
4617     nvlist_t *newnv)
4618 {
4619 	int i;
4620 	uint64_t val;
4621 	const char *names[] = {
4622 		ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
4623 		ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
4624 		ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
4625 		ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
4626 		ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
4627 		ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
4628 		ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
4629 		ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
4630 		ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
4631 		ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
4632 		ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
4633 	};
4634 	struct stat_array *nva;
4635 
4636 	unsigned int column_width = default_column_width(cb, IOS_LATENCY);
4637 	enum zfs_nicenum_format format;
4638 
4639 	nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);
4640 
4641 	if (cb->cb_literal)
4642 		format = ZFS_NICENUM_RAWTIME;
4643 	else
4644 		format = ZFS_NICENUM_TIME;
4645 
4646 	/* Print our avg latencies on the line */
4647 	for (i = 0; i < ARRAY_SIZE(names); i++) {
4648 		/* Compute average latency for a latency histo */
4649 		val = single_histo_average(nva[i].data, nva[i].count);
4650 		print_one_stat(val, format, column_width, cb->cb_scripted);
4651 	}
4652 	free_calc_stats(nva, ARRAY_SIZE(names));
4653 }
4654 
4655 /*
4656  * Print default statistics (capacity/operations/bandwidth)
4657  */
4658 static void
4659 print_iostat_default(vdev_stat_t *vs, iostat_cbdata_t *cb, double scale)
4660 {
4661 	unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
4662 	enum zfs_nicenum_format format;
4663 	char na;	/* char to print for "not applicable" values */
4664 
4665 	if (cb->cb_literal) {
4666 		format = ZFS_NICENUM_RAW;
4667 		na = '0';
4668 	} else {
4669 		format = ZFS_NICENUM_1024;
4670 		na = '-';
4671 	}
4672 
4673 	/* only toplevel vdevs have capacity stats */
4674 	if (vs->vs_space == 0) {
4675 		if (cb->cb_scripted)
4676 			printf("\t%c\t%c", na, na);
4677 		else
4678 			printf("  %*c  %*c", column_width, na, column_width,
4679 			    na);
4680 	} else {
4681 		print_one_stat(vs->vs_alloc, format, column_width,
4682 		    cb->cb_scripted);
4683 		print_one_stat(vs->vs_space - vs->vs_alloc, format,
4684 		    column_width, cb->cb_scripted);
4685 	}
4686 
4687 	print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
4688 	    format, column_width, cb->cb_scripted);
4689 	print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
4690 	    format, column_width, cb->cb_scripted);
4691 	print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
4692 	    format, column_width, cb->cb_scripted);
4693 	print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
4694 	    format, column_width, cb->cb_scripted);
4695 }
4696 
4697 static const char *class_name[] = {
4698 	VDEV_ALLOC_BIAS_DEDUP,
4699 	VDEV_ALLOC_BIAS_SPECIAL,
4700 	VDEV_ALLOC_CLASS_LOGS
4701 };
4702 
4703 /*
4704  * Print out all the statistics for the given vdev.  This can either be the
4705  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
4706  * is a verbose output, and we don't want to display the toplevel pool stats.
4707  *
4708  * Returns the number of stat lines printed.
4709  */
4710 static unsigned int
4711 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
4712     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
4713 {
4714 	nvlist_t **oldchild, **newchild;
4715 	uint_t c, children, oldchildren;
4716 	vdev_stat_t *oldvs, *newvs, *calcvs;
4717 	vdev_stat_t zerovs = { 0 };
4718 	char *vname;
4719 	int i;
4720 	int ret = 0;
4721 	uint64_t tdelta;
4722 	double scale;
4723 
4724 	if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
4725 		return (ret);
4726 
4727 	calcvs = safe_malloc(sizeof (*calcvs));
4728 
4729 	if (oldnv != NULL) {
4730 		verify(nvlist_lookup_uint64_array(oldnv,
4731 		    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
4732 	} else {
4733 		oldvs = &zerovs;
4734 	}
4735 
4736 	/* Do we only want to see a specific vdev? */
4737 	for (i = 0; i < cb->cb_vdevs.cb_names_count; i++) {
4738 		/* Yes we do.  Is this the vdev? */
4739 		if (strcmp(name, cb->cb_vdevs.cb_names[i]) == 0) {
4740 			/*
4741 			 * This is our vdev.  Since it is the only vdev we
4742 			 * will be displaying, make depth = 0 so that it
4743 			 * doesn't get indented.
4744 			 */
4745 			depth = 0;
4746 			break;
4747 		}
4748 	}
4749 
4750 	if (cb->cb_vdevs.cb_names_count && (i == cb->cb_vdevs.cb_names_count)) {
4751 		/* Couldn't match the name */
4752 		goto children;
4753 	}
4754 
4755 
4756 	verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
4757 	    (uint64_t **)&newvs, &c) == 0);
4758 
4759 	/*
4760 	 * Print the vdev name unless it's is a histogram.  Histograms
4761 	 * display the vdev name in the header itself.
4762 	 */
4763 	if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
4764 		if (cb->cb_scripted) {
4765 			printf("%s", name);
4766 		} else {
4767 			if (strlen(name) + depth > cb->cb_namewidth)
4768 				(void) printf("%*s%s", depth, "", name);
4769 			else
4770 				(void) printf("%*s%s%*s", depth, "", name,
4771 				    (int)(cb->cb_namewidth - strlen(name) -
4772 				    depth), "");
4773 		}
4774 	}
4775 
4776 	/* Calculate our scaling factor */
4777 	tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
4778 	if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
4779 		/*
4780 		 * If we specify printing histograms with no time interval, then
4781 		 * print the histogram numbers over the entire lifetime of the
4782 		 * vdev.
4783 		 */
4784 		scale = 1;
4785 	} else {
4786 		if (tdelta == 0)
4787 			scale = 1.0;
4788 		else
4789 			scale = (double)NANOSEC / tdelta;
4790 	}
4791 
4792 	if (cb->cb_flags & IOS_DEFAULT_M) {
4793 		calc_default_iostats(oldvs, newvs, calcvs);
4794 		print_iostat_default(calcvs, cb, scale);
4795 	}
4796 	if (cb->cb_flags & IOS_LATENCY_M)
4797 		print_iostat_latency(cb, oldnv, newnv);
4798 	if (cb->cb_flags & IOS_QUEUES_M)
4799 		print_iostat_queues(cb, oldnv, newnv);
4800 	if (cb->cb_flags & IOS_ANYHISTO_M) {
4801 		printf("\n");
4802 		print_iostat_histos(cb, oldnv, newnv, scale, name);
4803 	}
4804 
4805 	if (cb->vcdl != NULL) {
4806 		char *path;
4807 		if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
4808 		    &path) == 0) {
4809 			printf("  ");
4810 			zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
4811 		}
4812 	}
4813 
4814 	if (!(cb->cb_flags & IOS_ANYHISTO_M))
4815 		printf("\n");
4816 
4817 	ret++;
4818 
4819 children:
4820 
4821 	free(calcvs);
4822 
4823 	if (!cb->cb_verbose)
4824 		return (ret);
4825 
4826 	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
4827 	    &newchild, &children) != 0)
4828 		return (ret);
4829 
4830 	if (oldnv) {
4831 		if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
4832 		    &oldchild, &oldchildren) != 0)
4833 			return (ret);
4834 
4835 		children = MIN(oldchildren, children);
4836 	}
4837 
4838 	/*
4839 	 * print normal top-level devices
4840 	 */
4841 	for (c = 0; c < children; c++) {
4842 		uint64_t ishole = B_FALSE, islog = B_FALSE;
4843 
4844 		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
4845 		    &ishole);
4846 
4847 		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
4848 		    &islog);
4849 
4850 		if (ishole || islog)
4851 			continue;
4852 
4853 		if (nvlist_exists(newchild[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
4854 			continue;
4855 
4856 		vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4857 		    cb->cb_vdevs.cb_name_flags);
4858 		ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
4859 		    newchild[c], cb, depth + 2);
4860 		free(vname);
4861 	}
4862 
4863 	/*
4864 	 * print all other top-level devices
4865 	 */
4866 	for (uint_t n = 0; n < 3; n++) {
4867 		boolean_t printed = B_FALSE;
4868 
4869 		for (c = 0; c < children; c++) {
4870 			uint64_t islog = B_FALSE;
4871 			char *bias = NULL;
4872 			char *type = NULL;
4873 
4874 			(void) nvlist_lookup_uint64(newchild[c],
4875 			    ZPOOL_CONFIG_IS_LOG, &islog);
4876 			if (islog) {
4877 				bias = VDEV_ALLOC_CLASS_LOGS;
4878 			} else {
4879 				(void) nvlist_lookup_string(newchild[c],
4880 				    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
4881 				(void) nvlist_lookup_string(newchild[c],
4882 				    ZPOOL_CONFIG_TYPE, &type);
4883 			}
4884 			if (bias == NULL || strcmp(bias, class_name[n]) != 0)
4885 				continue;
4886 			if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
4887 				continue;
4888 
4889 			if (!printed) {
4890 				if ((!(cb->cb_flags & IOS_ANYHISTO_M)) &&
4891 				    !cb->cb_scripted &&
4892 				    !cb->cb_vdevs.cb_names) {
4893 					print_iostat_dashes(cb, 0,
4894 					    class_name[n]);
4895 				}
4896 				printf("\n");
4897 				printed = B_TRUE;
4898 			}
4899 
4900 			vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4901 			    cb->cb_vdevs.cb_name_flags);
4902 			ret += print_vdev_stats(zhp, vname, oldnv ?
4903 			    oldchild[c] : NULL, newchild[c], cb, depth + 2);
4904 			free(vname);
4905 		}
4906 	}
4907 
4908 	/*
4909 	 * Include level 2 ARC devices in iostat output
4910 	 */
4911 	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
4912 	    &newchild, &children) != 0)
4913 		return (ret);
4914 
4915 	if (oldnv) {
4916 		if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
4917 		    &oldchild, &oldchildren) != 0)
4918 			return (ret);
4919 
4920 		children = MIN(oldchildren, children);
4921 	}
4922 
4923 	if (children > 0) {
4924 		if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
4925 		    !cb->cb_vdevs.cb_names) {
4926 			print_iostat_dashes(cb, 0, "cache");
4927 		}
4928 		printf("\n");
4929 
4930 		for (c = 0; c < children; c++) {
4931 			vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4932 			    cb->cb_vdevs.cb_name_flags);
4933 			ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
4934 			    : NULL, newchild[c], cb, depth + 2);
4935 			free(vname);
4936 		}
4937 	}
4938 
4939 	return (ret);
4940 }
4941 
4942 static int
4943 refresh_iostat(zpool_handle_t *zhp, void *data)
4944 {
4945 	iostat_cbdata_t *cb = data;
4946 	boolean_t missing;
4947 
4948 	/*
4949 	 * If the pool has disappeared, remove it from the list and continue.
4950 	 */
4951 	if (zpool_refresh_stats(zhp, &missing) != 0)
4952 		return (-1);
4953 
4954 	if (missing)
4955 		pool_list_remove(cb->cb_list, zhp);
4956 
4957 	return (0);
4958 }
4959 
4960 /*
4961  * Callback to print out the iostats for the given pool.
4962  */
4963 static int
4964 print_iostat(zpool_handle_t *zhp, void *data)
4965 {
4966 	iostat_cbdata_t *cb = data;
4967 	nvlist_t *oldconfig, *newconfig;
4968 	nvlist_t *oldnvroot, *newnvroot;
4969 	int ret;
4970 
4971 	newconfig = zpool_get_config(zhp, &oldconfig);
4972 
4973 	if (cb->cb_iteration == 1)
4974 		oldconfig = NULL;
4975 
4976 	verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
4977 	    &newnvroot) == 0);
4978 
4979 	if (oldconfig == NULL)
4980 		oldnvroot = NULL;
4981 	else
4982 		verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
4983 		    &oldnvroot) == 0);
4984 
4985 	ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
4986 	    cb, 0);
4987 	if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
4988 	    !cb->cb_scripted && cb->cb_verbose &&
4989 	    !cb->cb_vdevs.cb_names_count) {
4990 		print_iostat_separator(cb);
4991 		if (cb->vcdl != NULL) {
4992 			print_cmd_columns(cb->vcdl, 1);
4993 		}
4994 		printf("\n");
4995 	}
4996 
4997 	return (ret);
4998 }
4999 
5000 static int
5001 get_columns(void)
5002 {
5003 	struct winsize ws;
5004 	int columns = 80;
5005 	int error;
5006 
5007 	if (isatty(STDOUT_FILENO)) {
5008 		error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
5009 		if (error == 0)
5010 			columns = ws.ws_col;
5011 	} else {
5012 		columns = 999;
5013 	}
5014 
5015 	return (columns);
5016 }
5017 
5018 /*
5019  * Return the required length of the pool/vdev name column.  The minimum
5020  * allowed width and output formatting flags must be provided.
5021  */
5022 static int
5023 get_namewidth(zpool_handle_t *zhp, int min_width, int flags, boolean_t verbose)
5024 {
5025 	nvlist_t *config, *nvroot;
5026 	int width = min_width;
5027 
5028 	if ((config = zpool_get_config(zhp, NULL)) != NULL) {
5029 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5030 		    &nvroot) == 0);
5031 		unsigned int poolname_len = strlen(zpool_get_name(zhp));
5032 		if (verbose == B_FALSE) {
5033 			width = MAX(poolname_len, min_width);
5034 		} else {
5035 			width = MAX(poolname_len,
5036 			    max_width(zhp, nvroot, 0, min_width, flags));
5037 		}
5038 	}
5039 
5040 	return (width);
5041 }
5042 
5043 /*
5044  * Parse the input string, get the 'interval' and 'count' value if there is one.
5045  */
5046 static void
5047 get_interval_count(int *argcp, char **argv, float *iv,
5048     unsigned long *cnt)
5049 {
5050 	float interval = 0;
5051 	unsigned long count = 0;
5052 	int argc = *argcp;
5053 
5054 	/*
5055 	 * Determine if the last argument is an integer or a pool name
5056 	 */
5057 	if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
5058 		char *end;
5059 
5060 		errno = 0;
5061 		interval = strtof(argv[argc - 1], &end);
5062 
5063 		if (*end == '\0' && errno == 0) {
5064 			if (interval == 0) {
5065 				(void) fprintf(stderr, gettext(
5066 				    "interval cannot be zero\n"));
5067 				usage(B_FALSE);
5068 			}
5069 			/*
5070 			 * Ignore the last parameter
5071 			 */
5072 			argc--;
5073 		} else {
5074 			/*
5075 			 * If this is not a valid number, just plow on.  The
5076 			 * user will get a more informative error message later
5077 			 * on.
5078 			 */
5079 			interval = 0;
5080 		}
5081 	}
5082 
5083 	/*
5084 	 * If the last argument is also an integer, then we have both a count
5085 	 * and an interval.
5086 	 */
5087 	if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
5088 		char *end;
5089 
5090 		errno = 0;
5091 		count = interval;
5092 		interval = strtof(argv[argc - 1], &end);
5093 
5094 		if (*end == '\0' && errno == 0) {
5095 			if (interval == 0) {
5096 				(void) fprintf(stderr, gettext(
5097 				    "interval cannot be zero\n"));
5098 				usage(B_FALSE);
5099 			}
5100 
5101 			/*
5102 			 * Ignore the last parameter
5103 			 */
5104 			argc--;
5105 		} else {
5106 			interval = 0;
5107 		}
5108 	}
5109 
5110 	*iv = interval;
5111 	*cnt = count;
5112 	*argcp = argc;
5113 }
5114 
5115 static void
5116 get_timestamp_arg(char c)
5117 {
5118 	if (c == 'u')
5119 		timestamp_fmt = UDATE;
5120 	else if (c == 'd')
5121 		timestamp_fmt = DDATE;
5122 	else
5123 		usage(B_FALSE);
5124 }
5125 
5126 /*
5127  * Return stat flags that are supported by all pools by both the module and
5128  * zpool iostat.  "*data" should be initialized to all 0xFFs before running.
5129  * It will get ANDed down until only the flags that are supported on all pools
5130  * remain.
5131  */
5132 static int
5133 get_stat_flags_cb(zpool_handle_t *zhp, void *data)
5134 {
5135 	uint64_t *mask = data;
5136 	nvlist_t *config, *nvroot, *nvx;
5137 	uint64_t flags = 0;
5138 	int i, j;
5139 
5140 	config = zpool_get_config(zhp, NULL);
5141 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5142 	    &nvroot) == 0);
5143 
5144 	/* Default stats are always supported, but for completeness.. */
5145 	if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
5146 		flags |= IOS_DEFAULT_M;
5147 
5148 	/* Get our extended stats nvlist from the main list */
5149 	if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
5150 	    &nvx) != 0) {
5151 		/*
5152 		 * No extended stats; they're probably running an older
5153 		 * module.  No big deal, we support that too.
5154 		 */
5155 		goto end;
5156 	}
5157 
5158 	/* For each extended stat, make sure all its nvpairs are supported */
5159 	for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
5160 		if (!vsx_type_to_nvlist[j][0])
5161 			continue;
5162 
5163 		/* Start off by assuming the flag is supported, then check */
5164 		flags |= (1ULL << j);
5165 		for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
5166 			if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
5167 				/* flag isn't supported */
5168 				flags = flags & ~(1ULL  << j);
5169 				break;
5170 			}
5171 		}
5172 	}
5173 end:
5174 	*mask = *mask & flags;
5175 	return (0);
5176 }
5177 
5178 /*
5179  * Return a bitmask of stats that are supported on all pools by both the module
5180  * and zpool iostat.
5181  */
5182 static uint64_t
5183 get_stat_flags(zpool_list_t *list)
5184 {
5185 	uint64_t mask = -1;
5186 
5187 	/*
5188 	 * get_stat_flags_cb() will lop off bits from "mask" until only the
5189 	 * flags that are supported on all pools remain.
5190 	 */
5191 	pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
5192 	return (mask);
5193 }
5194 
5195 /*
5196  * Return 1 if cb_data->cb_names[0] is this vdev's name, 0 otherwise.
5197  */
5198 static int
5199 is_vdev_cb(void *zhp_data, nvlist_t *nv, void *cb_data)
5200 {
5201 	vdev_cbdata_t *cb = cb_data;
5202 	char *name = NULL;
5203 	int ret = 1; /* assume match */
5204 	zpool_handle_t *zhp = zhp_data;
5205 
5206 	name = zpool_vdev_name(g_zfs, zhp, nv, cb->cb_name_flags);
5207 
5208 	if (strcmp(name, cb->cb_names[0])) {
5209 		free(name);
5210 		name = zpool_vdev_name(g_zfs, zhp, nv, VDEV_NAME_GUID);
5211 		ret = (strcmp(name, cb->cb_names[0]) == 0);
5212 	}
5213 	free(name);
5214 
5215 	return (ret);
5216 }
5217 
5218 /*
5219  * Returns 1 if cb_data->cb_names[0] is a vdev name, 0 otherwise.
5220  */
5221 static int
5222 is_vdev(zpool_handle_t *zhp, void *cb_data)
5223 {
5224 	return (for_each_vdev(zhp, is_vdev_cb, cb_data));
5225 }
5226 
5227 /*
5228  * Check if vdevs are in a pool
5229  *
5230  * Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
5231  * return 0.  If pool_name is NULL, then search all pools.
5232  */
5233 static int
5234 are_vdevs_in_pool(int argc, char **argv, char *pool_name,
5235     vdev_cbdata_t *cb)
5236 {
5237 	char **tmp_name;
5238 	int ret = 0;
5239 	int i;
5240 	int pool_count = 0;
5241 
5242 	if ((argc == 0) || !*argv)
5243 		return (0);
5244 
5245 	if (pool_name)
5246 		pool_count = 1;
5247 
5248 	/* Temporarily hijack cb_names for a second... */
5249 	tmp_name = cb->cb_names;
5250 
5251 	/* Go though our list of prospective vdev names */
5252 	for (i = 0; i < argc; i++) {
5253 		cb->cb_names = argv + i;
5254 
5255 		/* Is this name a vdev in our pools? */
5256 		ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
5257 		    ZFS_TYPE_POOL, B_FALSE, is_vdev, cb);
5258 		if (!ret) {
5259 			/* No match */
5260 			break;
5261 		}
5262 	}
5263 
5264 	cb->cb_names = tmp_name;
5265 
5266 	return (ret);
5267 }
5268 
5269 static int
5270 is_pool_cb(zpool_handle_t *zhp, void *data)
5271 {
5272 	char *name = data;
5273 	if (strcmp(name, zpool_get_name(zhp)) == 0)
5274 		return (1);
5275 
5276 	return (0);
5277 }
5278 
5279 /*
5280  * Do we have a pool named *name?  If so, return 1, otherwise 0.
5281  */
5282 static int
5283 is_pool(char *name)
5284 {
5285 	return (for_each_pool(0, NULL, B_TRUE, NULL, ZFS_TYPE_POOL, B_FALSE,
5286 	    is_pool_cb, name));
5287 }
5288 
5289 /* Are all our argv[] strings pool names?  If so return 1, 0 otherwise. */
5290 static int
5291 are_all_pools(int argc, char **argv)
5292 {
5293 	if ((argc == 0) || !*argv)
5294 		return (0);
5295 
5296 	while (--argc >= 0)
5297 		if (!is_pool(argv[argc]))
5298 			return (0);
5299 
5300 	return (1);
5301 }
5302 
5303 /*
5304  * Helper function to print out vdev/pool names we can't resolve.  Used for an
5305  * error message.
5306  */
5307 static void
5308 error_list_unresolved_vdevs(int argc, char **argv, char *pool_name,
5309     vdev_cbdata_t *cb)
5310 {
5311 	int i;
5312 	char *name;
5313 	char *str;
5314 	for (i = 0; i < argc; i++) {
5315 		name = argv[i];
5316 
5317 		if (is_pool(name))
5318 			str = gettext("pool");
5319 		else if (are_vdevs_in_pool(1, &name, pool_name, cb))
5320 			str = gettext("vdev in this pool");
5321 		else if (are_vdevs_in_pool(1, &name, NULL, cb))
5322 			str = gettext("vdev in another pool");
5323 		else
5324 			str = gettext("unknown");
5325 
5326 		fprintf(stderr, "\t%s (%s)\n", name, str);
5327 	}
5328 }
5329 
5330 /*
5331  * Same as get_interval_count(), but with additional checks to not misinterpret
5332  * guids as interval/count values.  Assumes VDEV_NAME_GUID is set in
5333  * cb.cb_vdevs.cb_name_flags.
5334  */
5335 static void
5336 get_interval_count_filter_guids(int *argc, char **argv, float *interval,
5337     unsigned long *count, iostat_cbdata_t *cb)
5338 {
5339 	char **tmpargv = argv;
5340 	int argc_for_interval = 0;
5341 
5342 	/* Is the last arg an interval value?  Or a guid? */
5343 	if (*argc >= 1 && !are_vdevs_in_pool(1, &argv[*argc - 1], NULL,
5344 	    &cb->cb_vdevs)) {
5345 		/*
5346 		 * The last arg is not a guid, so it's probably an
5347 		 * interval value.
5348 		 */
5349 		argc_for_interval++;
5350 
5351 		if (*argc >= 2 &&
5352 		    !are_vdevs_in_pool(1, &argv[*argc - 2], NULL,
5353 		    &cb->cb_vdevs)) {
5354 			/*
5355 			 * The 2nd to last arg is not a guid, so it's probably
5356 			 * an interval value.
5357 			 */
5358 			argc_for_interval++;
5359 		}
5360 	}
5361 
5362 	/* Point to our list of possible intervals */
5363 	tmpargv = &argv[*argc - argc_for_interval];
5364 
5365 	*argc = *argc - argc_for_interval;
5366 	get_interval_count(&argc_for_interval, tmpargv,
5367 	    interval, count);
5368 }
5369 
5370 /*
5371  * Floating point sleep().  Allows you to pass in a floating point value for
5372  * seconds.
5373  */
5374 static void
5375 fsleep(float sec)
5376 {
5377 	struct timespec req;
5378 	req.tv_sec = floor(sec);
5379 	req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
5380 	nanosleep(&req, NULL);
5381 }
5382 
5383 /*
5384  * Terminal height, in rows. Returns -1 if stdout is not connected to a TTY or
5385  * if we were unable to determine its size.
5386  */
5387 static int
5388 terminal_height(void)
5389 {
5390 	struct winsize win;
5391 
5392 	if (isatty(STDOUT_FILENO) == 0)
5393 		return (-1);
5394 
5395 	if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) != -1 && win.ws_row > 0)
5396 		return (win.ws_row);
5397 
5398 	return (-1);
5399 }
5400 
5401 /*
5402  * Run one of the zpool status/iostat -c scripts with the help (-h) option and
5403  * print the result.
5404  *
5405  * name:	Short name of the script ('iostat').
5406  * path:	Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
5407  */
5408 static void
5409 print_zpool_script_help(char *name, char *path)
5410 {
5411 	char *argv[] = {path, "-h", NULL};
5412 	char **lines = NULL;
5413 	int lines_cnt = 0;
5414 	int rc;
5415 
5416 	rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
5417 	    &lines_cnt);
5418 	if (rc != 0 || lines == NULL || lines_cnt <= 0) {
5419 		if (lines != NULL)
5420 			libzfs_free_str_array(lines, lines_cnt);
5421 		return;
5422 	}
5423 
5424 	for (int i = 0; i < lines_cnt; i++)
5425 		if (!is_blank_str(lines[i]))
5426 			printf("  %-14s  %s\n", name, lines[i]);
5427 
5428 	libzfs_free_str_array(lines, lines_cnt);
5429 }
5430 
5431 /*
5432  * Go though the zpool status/iostat -c scripts in the user's path, run their
5433  * help option (-h), and print out the results.
5434  */
5435 static void
5436 print_zpool_dir_scripts(char *dirpath)
5437 {
5438 	DIR *dir;
5439 	struct dirent *ent;
5440 	char fullpath[MAXPATHLEN];
5441 	struct stat dir_stat;
5442 
5443 	if ((dir = opendir(dirpath)) != NULL) {
5444 		/* print all the files and directories within directory */
5445 		while ((ent = readdir(dir)) != NULL) {
5446 			sprintf(fullpath, "%s/%s", dirpath, ent->d_name);
5447 
5448 			/* Print the scripts */
5449 			if (stat(fullpath, &dir_stat) == 0)
5450 				if (dir_stat.st_mode & S_IXUSR &&
5451 				    S_ISREG(dir_stat.st_mode))
5452 					print_zpool_script_help(ent->d_name,
5453 					    fullpath);
5454 		}
5455 		closedir(dir);
5456 	}
5457 }
5458 
5459 /*
5460  * Print out help text for all zpool status/iostat -c scripts.
5461  */
5462 static void
5463 print_zpool_script_list(char *subcommand)
5464 {
5465 	char *dir, *sp, *tmp;
5466 
5467 	printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);
5468 
5469 	sp = zpool_get_cmd_search_path();
5470 	if (sp == NULL)
5471 		return;
5472 
5473 	for (dir = strtok_r(sp, ":", &tmp);
5474 	    dir != NULL;
5475 	    dir = strtok_r(NULL, ":", &tmp))
5476 		print_zpool_dir_scripts(dir);
5477 
5478 	free(sp);
5479 }
5480 
5481 /*
5482  * Set the minimum pool/vdev name column width.  The width must be at least 10,
5483  * but may be as large as the column width - 42 so it still fits on one line.
5484  * NOTE: 42 is the width of the default capacity/operations/bandwidth output
5485  */
5486 static int
5487 get_namewidth_iostat(zpool_handle_t *zhp, void *data)
5488 {
5489 	iostat_cbdata_t *cb = data;
5490 	int width, available_width;
5491 
5492 	/*
5493 	 * get_namewidth() returns the maximum width of any name in that column
5494 	 * for any pool/vdev/device line that will be output.
5495 	 */
5496 	width = get_namewidth(zhp, cb->cb_namewidth, cb->cb_vdevs.cb_name_flags,
5497 	    cb->cb_verbose);
5498 
5499 	/*
5500 	 * The width we are calculating is the width of the header and also the
5501 	 * padding width for names that are less than maximum width.  The stats
5502 	 * take up 42 characters, so the width available for names is:
5503 	 */
5504 	available_width = get_columns() - 42;
5505 
5506 	/*
5507 	 * If the maximum width fits on a screen, then great!  Make everything
5508 	 * line up by justifying all lines to the same width.  If that max
5509 	 * width is larger than what's available, the name plus stats won't fit
5510 	 * on one line, and justifying to that width would cause every line to
5511 	 * wrap on the screen.  We only want lines with long names to wrap.
5512 	 * Limit the padding to what won't wrap.
5513 	 */
5514 	if (width > available_width)
5515 		width = available_width;
5516 
5517 	/*
5518 	 * And regardless of whatever the screen width is (get_columns can
5519 	 * return 0 if the width is not known or less than 42 for a narrow
5520 	 * terminal) have the width be a minimum of 10.
5521 	 */
5522 	if (width < 10)
5523 		width = 10;
5524 
5525 	/* Save the calculated width */
5526 	cb->cb_namewidth = width;
5527 
5528 	return (0);
5529 }
5530 
5531 /*
5532  * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
5533  *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
5534  *              [interval [count]]
5535  *
5536  *	-c CMD  For each vdev, run command CMD
5537  *	-g	Display guid for individual vdev name.
5538  *	-L	Follow links when resolving vdev path name.
5539  *	-P	Display full path for vdev name.
5540  *	-v	Display statistics for individual vdevs
5541  *	-h	Display help
5542  *	-p	Display values in parsable (exact) format.
5543  *	-H	Scripted mode.  Don't display headers, and separate properties
5544  *		by a single tab.
5545  *	-l	Display average latency
5546  *	-q	Display queue depths
5547  *	-w	Display latency histograms
5548  *	-r	Display request size histogram
5549  *	-T	Display a timestamp in date(1) or Unix format
5550  *	-n	Only print headers once
5551  *
5552  * This command can be tricky because we want to be able to deal with pool
5553  * creation/destruction as well as vdev configuration changes.  The bulk of this
5554  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
5555  * on pool_list_update() to detect the addition of new pools.  Configuration
5556  * changes are all handled within libzfs.
5557  */
5558 int
5559 zpool_do_iostat(int argc, char **argv)
5560 {
5561 	int c;
5562 	int ret;
5563 	int npools;
5564 	float interval = 0;
5565 	unsigned long count = 0;
5566 	int winheight = 24;
5567 	zpool_list_t *list;
5568 	boolean_t verbose = B_FALSE;
5569 	boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
5570 	boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE;
5571 	boolean_t omit_since_boot = B_FALSE;
5572 	boolean_t guid = B_FALSE;
5573 	boolean_t follow_links = B_FALSE;
5574 	boolean_t full_name = B_FALSE;
5575 	boolean_t headers_once = B_FALSE;
5576 	iostat_cbdata_t cb = { 0 };
5577 	char *cmd = NULL;
5578 
5579 	/* Used for printing error message */
5580 	const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
5581 	    [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};
5582 
5583 	uint64_t unsupported_flags;
5584 
5585 	/* check options */
5586 	while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwnH")) != -1) {
5587 		switch (c) {
5588 		case 'c':
5589 			if (cmd != NULL) {
5590 				fprintf(stderr,
5591 				    gettext("Can't set -c flag twice\n"));
5592 				exit(1);
5593 			}
5594 
5595 			if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
5596 			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
5597 				fprintf(stderr, gettext(
5598 				    "Can't run -c, disabled by "
5599 				    "ZPOOL_SCRIPTS_ENABLED.\n"));
5600 				exit(1);
5601 			}
5602 
5603 			if ((getuid() <= 0 || geteuid() <= 0) &&
5604 			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
5605 				fprintf(stderr, gettext(
5606 				    "Can't run -c with root privileges "
5607 				    "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
5608 				exit(1);
5609 			}
5610 			cmd = optarg;
5611 			verbose = B_TRUE;
5612 			break;
5613 		case 'g':
5614 			guid = B_TRUE;
5615 			break;
5616 		case 'L':
5617 			follow_links = B_TRUE;
5618 			break;
5619 		case 'P':
5620 			full_name = B_TRUE;
5621 			break;
5622 		case 'T':
5623 			get_timestamp_arg(*optarg);
5624 			break;
5625 		case 'v':
5626 			verbose = B_TRUE;
5627 			break;
5628 		case 'p':
5629 			parsable = B_TRUE;
5630 			break;
5631 		case 'l':
5632 			latency = B_TRUE;
5633 			break;
5634 		case 'q':
5635 			queues = B_TRUE;
5636 			break;
5637 		case 'H':
5638 			scripted = B_TRUE;
5639 			break;
5640 		case 'w':
5641 			l_histo = B_TRUE;
5642 			break;
5643 		case 'r':
5644 			rq_histo = B_TRUE;
5645 			break;
5646 		case 'y':
5647 			omit_since_boot = B_TRUE;
5648 			break;
5649 		case 'n':
5650 			headers_once = B_TRUE;
5651 			break;
5652 		case 'h':
5653 			usage(B_FALSE);
5654 			break;
5655 		case '?':
5656 			if (optopt == 'c') {
5657 				print_zpool_script_list("iostat");
5658 				exit(0);
5659 			} else {
5660 				fprintf(stderr,
5661 				    gettext("invalid option '%c'\n"), optopt);
5662 			}
5663 			usage(B_FALSE);
5664 		}
5665 	}
5666 
5667 	argc -= optind;
5668 	argv += optind;
5669 
5670 	cb.cb_literal = parsable;
5671 	cb.cb_scripted = scripted;
5672 
5673 	if (guid)
5674 		cb.cb_vdevs.cb_name_flags |= VDEV_NAME_GUID;
5675 	if (follow_links)
5676 		cb.cb_vdevs.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
5677 	if (full_name)
5678 		cb.cb_vdevs.cb_name_flags |= VDEV_NAME_PATH;
5679 	cb.cb_iteration = 0;
5680 	cb.cb_namewidth = 0;
5681 	cb.cb_verbose = verbose;
5682 
5683 	/* Get our interval and count values (if any) */
5684 	if (guid) {
5685 		get_interval_count_filter_guids(&argc, argv, &interval,
5686 		    &count, &cb);
5687 	} else {
5688 		get_interval_count(&argc, argv, &interval, &count);
5689 	}
5690 
5691 	if (argc == 0) {
5692 		/* No args, so just print the defaults. */
5693 	} else if (are_all_pools(argc, argv)) {
5694 		/* All the args are pool names */
5695 	} else if (are_vdevs_in_pool(argc, argv, NULL, &cb.cb_vdevs)) {
5696 		/* All the args are vdevs */
5697 		cb.cb_vdevs.cb_names = argv;
5698 		cb.cb_vdevs.cb_names_count = argc;
5699 		argc = 0; /* No pools to process */
5700 	} else if (are_all_pools(1, argv)) {
5701 		/* The first arg is a pool name */
5702 		if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
5703 		    &cb.cb_vdevs)) {
5704 			/* ...and the rest are vdev names */
5705 			cb.cb_vdevs.cb_names = argv + 1;
5706 			cb.cb_vdevs.cb_names_count = argc - 1;
5707 			argc = 1; /* One pool to process */
5708 		} else {
5709 			fprintf(stderr, gettext("Expected either a list of "));
5710 			fprintf(stderr, gettext("pools, or list of vdevs in"));
5711 			fprintf(stderr, " \"%s\", ", argv[0]);
5712 			fprintf(stderr, gettext("but got:\n"));
5713 			error_list_unresolved_vdevs(argc - 1, argv + 1,
5714 			    argv[0], &cb.cb_vdevs);
5715 			fprintf(stderr, "\n");
5716 			usage(B_FALSE);
5717 			return (1);
5718 		}
5719 	} else {
5720 		/*
5721 		 * The args don't make sense. The first arg isn't a pool name,
5722 		 * nor are all the args vdevs.
5723 		 */
5724 		fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
5725 		fprintf(stderr, "\n");
5726 		return (1);
5727 	}
5728 
5729 	if (cb.cb_vdevs.cb_names_count != 0) {
5730 		/*
5731 		 * If user specified vdevs, it implies verbose.
5732 		 */
5733 		cb.cb_verbose = B_TRUE;
5734 	}
5735 
5736 	/*
5737 	 * Construct the list of all interesting pools.
5738 	 */
5739 	ret = 0;
5740 	if ((list = pool_list_get(argc, argv, NULL, ZFS_TYPE_POOL, parsable,
5741 	    &ret)) == NULL)
5742 		return (1);
5743 
5744 	if (pool_list_count(list) == 0 && argc != 0) {
5745 		pool_list_free(list);
5746 		return (1);
5747 	}
5748 
5749 	if (pool_list_count(list) == 0 && interval == 0) {
5750 		pool_list_free(list);
5751 		(void) fprintf(stderr, gettext("no pools available\n"));
5752 		return (1);
5753 	}
5754 
5755 	if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
5756 		pool_list_free(list);
5757 		(void) fprintf(stderr,
5758 		    gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
5759 		usage(B_FALSE);
5760 		return (1);
5761 	}
5762 
5763 	if (l_histo && rq_histo) {
5764 		pool_list_free(list);
5765 		(void) fprintf(stderr,
5766 		    gettext("Only one of [-r|-w] can be passed at a time\n"));
5767 		usage(B_FALSE);
5768 		return (1);
5769 	}
5770 
5771 	/*
5772 	 * Enter the main iostat loop.
5773 	 */
5774 	cb.cb_list = list;
5775 
5776 	if (l_histo) {
5777 		/*
5778 		 * Histograms tables look out of place when you try to display
5779 		 * them with the other stats, so make a rule that you can only
5780 		 * print histograms by themselves.
5781 		 */
5782 		cb.cb_flags = IOS_L_HISTO_M;
5783 	} else if (rq_histo) {
5784 		cb.cb_flags = IOS_RQ_HISTO_M;
5785 	} else {
5786 		cb.cb_flags = IOS_DEFAULT_M;
5787 		if (latency)
5788 			cb.cb_flags |= IOS_LATENCY_M;
5789 		if (queues)
5790 			cb.cb_flags |= IOS_QUEUES_M;
5791 	}
5792 
5793 	/*
5794 	 * See if the module supports all the stats we want to display.
5795 	 */
5796 	unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
5797 	if (unsupported_flags) {
5798 		uint64_t f;
5799 		int idx;
5800 		fprintf(stderr,
5801 		    gettext("The loaded zfs module doesn't support:"));
5802 
5803 		/* for each bit set in unsupported_flags */
5804 		for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
5805 			idx = lowbit64(f) - 1;
5806 			fprintf(stderr, " -%c", flag_to_arg[idx]);
5807 		}
5808 
5809 		fprintf(stderr, ".  Try running a newer module.\n");
5810 		pool_list_free(list);
5811 
5812 		return (1);
5813 	}
5814 
5815 	for (;;) {
5816 		if ((npools = pool_list_count(list)) == 0)
5817 			(void) fprintf(stderr, gettext("no pools available\n"));
5818 		else {
5819 			/*
5820 			 * If this is the first iteration and -y was supplied
5821 			 * we skip any printing.
5822 			 */
5823 			boolean_t skip = (omit_since_boot &&
5824 			    cb.cb_iteration == 0);
5825 
5826 			/*
5827 			 * Refresh all statistics.  This is done as an
5828 			 * explicit step before calculating the maximum name
5829 			 * width, so that any * configuration changes are
5830 			 * properly accounted for.
5831 			 */
5832 			(void) pool_list_iter(list, B_FALSE, refresh_iostat,
5833 			    &cb);
5834 
5835 			/*
5836 			 * Iterate over all pools to determine the maximum width
5837 			 * for the pool / device name column across all pools.
5838 			 */
5839 			cb.cb_namewidth = 0;
5840 			(void) pool_list_iter(list, B_FALSE,
5841 			    get_namewidth_iostat, &cb);
5842 
5843 			if (timestamp_fmt != NODATE)
5844 				print_timestamp(timestamp_fmt);
5845 
5846 			if (cmd != NULL && cb.cb_verbose &&
5847 			    !(cb.cb_flags & IOS_ANYHISTO_M)) {
5848 				cb.vcdl = all_pools_for_each_vdev_run(argc,
5849 				    argv, cmd, g_zfs, cb.cb_vdevs.cb_names,
5850 				    cb.cb_vdevs.cb_names_count,
5851 				    cb.cb_vdevs.cb_name_flags);
5852 			} else {
5853 				cb.vcdl = NULL;
5854 			}
5855 
5856 
5857 			/*
5858 			 * Check terminal size so we can print headers
5859 			 * even when terminal window has its height
5860 			 * changed.
5861 			 */
5862 			winheight = terminal_height();
5863 			/*
5864 			 * Are we connected to TTY? If not, headers_once
5865 			 * should be true, to avoid breaking scripts.
5866 			 */
5867 			if (winheight < 0)
5868 				headers_once = B_TRUE;
5869 
5870 			/*
5871 			 * If it's the first time and we're not skipping it,
5872 			 * or either skip or verbose mode, print the header.
5873 			 *
5874 			 * The histogram code explicitly prints its header on
5875 			 * every vdev, so skip this for histograms.
5876 			 */
5877 			if (((++cb.cb_iteration == 1 && !skip) ||
5878 			    (skip != verbose) ||
5879 			    (!headers_once &&
5880 			    (cb.cb_iteration % winheight) == 0)) &&
5881 			    (!(cb.cb_flags & IOS_ANYHISTO_M)) &&
5882 			    !cb.cb_scripted)
5883 				print_iostat_header(&cb);
5884 
5885 			if (skip) {
5886 				(void) fsleep(interval);
5887 				continue;
5888 			}
5889 
5890 			pool_list_iter(list, B_FALSE, print_iostat, &cb);
5891 
5892 			/*
5893 			 * If there's more than one pool, and we're not in
5894 			 * verbose mode (which prints a separator for us),
5895 			 * then print a separator.
5896 			 *
5897 			 * In addition, if we're printing specific vdevs then
5898 			 * we also want an ending separator.
5899 			 */
5900 			if (((npools > 1 && !verbose &&
5901 			    !(cb.cb_flags & IOS_ANYHISTO_M)) ||
5902 			    (!(cb.cb_flags & IOS_ANYHISTO_M) &&
5903 			    cb.cb_vdevs.cb_names_count)) &&
5904 			    !cb.cb_scripted) {
5905 				print_iostat_separator(&cb);
5906 				if (cb.vcdl != NULL)
5907 					print_cmd_columns(cb.vcdl, 1);
5908 				printf("\n");
5909 			}
5910 
5911 			if (cb.vcdl != NULL)
5912 				free_vdev_cmd_data_list(cb.vcdl);
5913 
5914 		}
5915 
5916 		/*
5917 		 * Flush the output so that redirection to a file isn't buffered
5918 		 * indefinitely.
5919 		 */
5920 		(void) fflush(stdout);
5921 
5922 		if (interval == 0)
5923 			break;
5924 
5925 		if (count != 0 && --count == 0)
5926 			break;
5927 
5928 		(void) fsleep(interval);
5929 	}
5930 
5931 	pool_list_free(list);
5932 
5933 	return (ret);
5934 }
5935 
5936 typedef struct list_cbdata {
5937 	boolean_t	cb_verbose;
5938 	int		cb_name_flags;
5939 	int		cb_namewidth;
5940 	boolean_t	cb_scripted;
5941 	zprop_list_t	*cb_proplist;
5942 	boolean_t	cb_literal;
5943 } list_cbdata_t;
5944 
5945 
5946 /*
5947  * Given a list of columns to display, output appropriate headers for each one.
5948  */
5949 static void
5950 print_header(list_cbdata_t *cb)
5951 {
5952 	zprop_list_t *pl = cb->cb_proplist;
5953 	char headerbuf[ZPOOL_MAXPROPLEN];
5954 	const char *header;
5955 	boolean_t first = B_TRUE;
5956 	boolean_t right_justify;
5957 	size_t width = 0;
5958 
5959 	for (; pl != NULL; pl = pl->pl_next) {
5960 		width = pl->pl_width;
5961 		if (first && cb->cb_verbose) {
5962 			/*
5963 			 * Reset the width to accommodate the verbose listing
5964 			 * of devices.
5965 			 */
5966 			width = cb->cb_namewidth;
5967 		}
5968 
5969 		if (!first)
5970 			(void) printf("  ");
5971 		else
5972 			first = B_FALSE;
5973 
5974 		right_justify = B_FALSE;
5975 		if (pl->pl_prop != ZPROP_INVAL) {
5976 			header = zpool_prop_column_name(pl->pl_prop);
5977 			right_justify = zpool_prop_align_right(pl->pl_prop);
5978 		} else {
5979 			int i;
5980 
5981 			for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
5982 				headerbuf[i] = toupper(pl->pl_user_prop[i]);
5983 			headerbuf[i] = '\0';
5984 			header = headerbuf;
5985 		}
5986 
5987 		if (pl->pl_next == NULL && !right_justify)
5988 			(void) printf("%s", header);
5989 		else if (right_justify)
5990 			(void) printf("%*s", (int)width, header);
5991 		else
5992 			(void) printf("%-*s", (int)width, header);
5993 	}
5994 
5995 	(void) printf("\n");
5996 }
5997 
5998 /*
5999  * Given a pool and a list of properties, print out all the properties according
6000  * to the described layout. Used by zpool_do_list().
6001  */
6002 static void
6003 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
6004 {
6005 	zprop_list_t *pl = cb->cb_proplist;
6006 	boolean_t first = B_TRUE;
6007 	char property[ZPOOL_MAXPROPLEN];
6008 	char *propstr;
6009 	boolean_t right_justify;
6010 	size_t width;
6011 
6012 	for (; pl != NULL; pl = pl->pl_next) {
6013 
6014 		width = pl->pl_width;
6015 		if (first && cb->cb_verbose) {
6016 			/*
6017 			 * Reset the width to accommodate the verbose listing
6018 			 * of devices.
6019 			 */
6020 			width = cb->cb_namewidth;
6021 		}
6022 
6023 		if (!first) {
6024 			if (cb->cb_scripted)
6025 				(void) printf("\t");
6026 			else
6027 				(void) printf("  ");
6028 		} else {
6029 			first = B_FALSE;
6030 		}
6031 
6032 		right_justify = B_FALSE;
6033 		if (pl->pl_prop != ZPROP_INVAL) {
6034 			if (zpool_get_prop(zhp, pl->pl_prop, property,
6035 			    sizeof (property), NULL, cb->cb_literal) != 0)
6036 				propstr = "-";
6037 			else
6038 				propstr = property;
6039 
6040 			right_justify = zpool_prop_align_right(pl->pl_prop);
6041 		} else if ((zpool_prop_feature(pl->pl_user_prop) ||
6042 		    zpool_prop_unsupported(pl->pl_user_prop)) &&
6043 		    zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
6044 		    sizeof (property)) == 0) {
6045 			propstr = property;
6046 		} else {
6047 			propstr = "-";
6048 		}
6049 
6050 
6051 		/*
6052 		 * If this is being called in scripted mode, or if this is the
6053 		 * last column and it is left-justified, don't include a width
6054 		 * format specifier.
6055 		 */
6056 		if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
6057 			(void) printf("%s", propstr);
6058 		else if (right_justify)
6059 			(void) printf("%*s", (int)width, propstr);
6060 		else
6061 			(void) printf("%-*s", (int)width, propstr);
6062 	}
6063 
6064 	(void) printf("\n");
6065 }
6066 
6067 static void
6068 print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
6069     boolean_t scripted, boolean_t valid, enum zfs_nicenum_format format)
6070 {
6071 	char propval[64];
6072 	boolean_t fixed;
6073 	size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
6074 
6075 	switch (prop) {
6076 	case ZPOOL_PROP_EXPANDSZ:
6077 	case ZPOOL_PROP_CHECKPOINT:
6078 	case ZPOOL_PROP_DEDUPRATIO:
6079 		if (value == 0)
6080 			(void) strlcpy(propval, "-", sizeof (propval));
6081 		else
6082 			zfs_nicenum_format(value, propval, sizeof (propval),
6083 			    format);
6084 		break;
6085 	case ZPOOL_PROP_FRAGMENTATION:
6086 		if (value == ZFS_FRAG_INVALID) {
6087 			(void) strlcpy(propval, "-", sizeof (propval));
6088 		} else if (format == ZFS_NICENUM_RAW) {
6089 			(void) snprintf(propval, sizeof (propval), "%llu",
6090 			    (unsigned long long)value);
6091 		} else {
6092 			(void) snprintf(propval, sizeof (propval), "%llu%%",
6093 			    (unsigned long long)value);
6094 		}
6095 		break;
6096 	case ZPOOL_PROP_CAPACITY:
6097 		/* capacity value is in parts-per-10,000 (aka permyriad) */
6098 		if (format == ZFS_NICENUM_RAW)
6099 			(void) snprintf(propval, sizeof (propval), "%llu",
6100 			    (unsigned long long)value / 100);
6101 		else
6102 			(void) snprintf(propval, sizeof (propval),
6103 			    value < 1000 ? "%1.2f%%" : value < 10000 ?
6104 			    "%2.1f%%" : "%3.0f%%", value / 100.0);
6105 		break;
6106 	case ZPOOL_PROP_HEALTH:
6107 		width = 8;
6108 		(void) strlcpy(propval, str, sizeof (propval));
6109 		break;
6110 	default:
6111 		zfs_nicenum_format(value, propval, sizeof (propval), format);
6112 	}
6113 
6114 	if (!valid)
6115 		(void) strlcpy(propval, "-", sizeof (propval));
6116 
6117 	if (scripted)
6118 		(void) printf("\t%s", propval);
6119 	else
6120 		(void) printf("  %*s", (int)width, propval);
6121 }
6122 
6123 /*
6124  * print static default line per vdev
6125  * not compatible with '-o' <proplist> option
6126  */
6127 static void
6128 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
6129     list_cbdata_t *cb, int depth, boolean_t isspare)
6130 {
6131 	nvlist_t **child;
6132 	vdev_stat_t *vs;
6133 	uint_t c, children;
6134 	char *vname;
6135 	boolean_t scripted = cb->cb_scripted;
6136 	uint64_t islog = B_FALSE;
6137 	char *dashes = "%-*s      -      -      -        -         "
6138 	    "-      -      -      -  -\n";
6139 
6140 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
6141 	    (uint64_t **)&vs, &c) == 0);
6142 
6143 	if (name != NULL) {
6144 		boolean_t toplevel = (vs->vs_space != 0);
6145 		uint64_t cap;
6146 		enum zfs_nicenum_format format;
6147 		const char *state;
6148 
6149 		if (cb->cb_literal)
6150 			format = ZFS_NICENUM_RAW;
6151 		else
6152 			format = ZFS_NICENUM_1024;
6153 
6154 		if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
6155 			return;
6156 
6157 		if (scripted)
6158 			(void) printf("\t%s", name);
6159 		else if (strlen(name) + depth > cb->cb_namewidth)
6160 			(void) printf("%*s%s", depth, "", name);
6161 		else
6162 			(void) printf("%*s%s%*s", depth, "", name,
6163 			    (int)(cb->cb_namewidth - strlen(name) - depth), "");
6164 
6165 		/*
6166 		 * Print the properties for the individual vdevs. Some
6167 		 * properties are only applicable to toplevel vdevs. The
6168 		 * 'toplevel' boolean value is passed to the print_one_column()
6169 		 * to indicate that the value is valid.
6170 		 */
6171 		print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, NULL, scripted,
6172 		    toplevel, format);
6173 		print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, NULL,
6174 		    scripted, toplevel, format);
6175 		print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
6176 		    NULL, scripted, toplevel, format);
6177 		print_one_column(ZPOOL_PROP_CHECKPOINT,
6178 		    vs->vs_checkpoint_space, NULL, scripted, toplevel, format);
6179 		print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, NULL,
6180 		    scripted, B_TRUE, format);
6181 		print_one_column(ZPOOL_PROP_FRAGMENTATION,
6182 		    vs->vs_fragmentation, NULL, scripted,
6183 		    (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
6184 		    format);
6185 		cap = (vs->vs_space == 0) ? 0 :
6186 		    (vs->vs_alloc * 10000 / vs->vs_space);
6187 		print_one_column(ZPOOL_PROP_CAPACITY, cap, NULL,
6188 		    scripted, toplevel, format);
6189 		print_one_column(ZPOOL_PROP_DEDUPRATIO, 0, NULL,
6190 		    scripted, toplevel, format);
6191 		state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
6192 		if (isspare) {
6193 			if (vs->vs_aux == VDEV_AUX_SPARED)
6194 				state = "INUSE";
6195 			else if (vs->vs_state == VDEV_STATE_HEALTHY)
6196 				state = "AVAIL";
6197 		}
6198 		print_one_column(ZPOOL_PROP_HEALTH, 0, state, scripted,
6199 		    B_TRUE, format);
6200 		(void) printf("\n");
6201 	}
6202 
6203 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
6204 	    &child, &children) != 0)
6205 		return;
6206 
6207 	/* list the normal vdevs first */
6208 	for (c = 0; c < children; c++) {
6209 		uint64_t ishole = B_FALSE;
6210 
6211 		if (nvlist_lookup_uint64(child[c],
6212 		    ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
6213 			continue;
6214 
6215 		if (nvlist_lookup_uint64(child[c],
6216 		    ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog)
6217 			continue;
6218 
6219 		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
6220 			continue;
6221 
6222 		vname = zpool_vdev_name(g_zfs, zhp, child[c],
6223 		    cb->cb_name_flags);
6224 		print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE);
6225 		free(vname);
6226 	}
6227 
6228 	/* list the classes: 'logs', 'dedup', and 'special' */
6229 	for (uint_t n = 0; n < 3; n++) {
6230 		boolean_t printed = B_FALSE;
6231 
6232 		for (c = 0; c < children; c++) {
6233 			char *bias = NULL;
6234 			char *type = NULL;
6235 
6236 			if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
6237 			    &islog) == 0 && islog) {
6238 				bias = VDEV_ALLOC_CLASS_LOGS;
6239 			} else {
6240 				(void) nvlist_lookup_string(child[c],
6241 				    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
6242 				(void) nvlist_lookup_string(child[c],
6243 				    ZPOOL_CONFIG_TYPE, &type);
6244 			}
6245 			if (bias == NULL || strcmp(bias, class_name[n]) != 0)
6246 				continue;
6247 			if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
6248 				continue;
6249 
6250 			if (!printed) {
6251 				/* LINTED E_SEC_PRINTF_VAR_FMT */
6252 				(void) printf(dashes, cb->cb_namewidth,
6253 				    class_name[n]);
6254 				printed = B_TRUE;
6255 			}
6256 			vname = zpool_vdev_name(g_zfs, zhp, child[c],
6257 			    cb->cb_name_flags);
6258 			print_list_stats(zhp, vname, child[c], cb, depth + 2,
6259 			    B_FALSE);
6260 			free(vname);
6261 		}
6262 	}
6263 
6264 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
6265 	    &child, &children) == 0 && children > 0) {
6266 		/* LINTED E_SEC_PRINTF_VAR_FMT */
6267 		(void) printf(dashes, cb->cb_namewidth, "cache");
6268 		for (c = 0; c < children; c++) {
6269 			vname = zpool_vdev_name(g_zfs, zhp, child[c],
6270 			    cb->cb_name_flags);
6271 			print_list_stats(zhp, vname, child[c], cb, depth + 2,
6272 			    B_FALSE);
6273 			free(vname);
6274 		}
6275 	}
6276 
6277 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
6278 	    &children) == 0 && children > 0) {
6279 		/* LINTED E_SEC_PRINTF_VAR_FMT */
6280 		(void) printf(dashes, cb->cb_namewidth, "spare");
6281 		for (c = 0; c < children; c++) {
6282 			vname = zpool_vdev_name(g_zfs, zhp, child[c],
6283 			    cb->cb_name_flags);
6284 			print_list_stats(zhp, vname, child[c], cb, depth + 2,
6285 			    B_TRUE);
6286 			free(vname);
6287 		}
6288 	}
6289 }
6290 
6291 /*
6292  * Generic callback function to list a pool.
6293  */
6294 static int
6295 list_callback(zpool_handle_t *zhp, void *data)
6296 {
6297 	list_cbdata_t *cbp = data;
6298 
6299 	print_pool(zhp, cbp);
6300 
6301 	if (cbp->cb_verbose) {
6302 		nvlist_t *config, *nvroot;
6303 
6304 		config = zpool_get_config(zhp, NULL);
6305 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
6306 		    &nvroot) == 0);
6307 		print_list_stats(zhp, NULL, nvroot, cbp, 0, B_FALSE);
6308 	}
6309 
6310 	return (0);
6311 }
6312 
6313 /*
6314  * Set the minimum pool/vdev name column width.  The width must be at least 9,
6315  * but may be as large as needed.
6316  */
6317 static int
6318 get_namewidth_list(zpool_handle_t *zhp, void *data)
6319 {
6320 	list_cbdata_t *cb = data;
6321 	int width;
6322 
6323 	width = get_namewidth(zhp, cb->cb_namewidth, cb->cb_name_flags,
6324 	    cb->cb_verbose);
6325 
6326 	if (width < 9)
6327 		width = 9;
6328 
6329 	cb->cb_namewidth = width;
6330 
6331 	return (0);
6332 }
6333 
6334 /*
6335  * zpool list [-gHLpP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
6336  *
6337  *	-g	Display guid for individual vdev name.
6338  *	-H	Scripted mode.  Don't display headers, and separate properties
6339  *		by a single tab.
6340  *	-L	Follow links when resolving vdev path name.
6341  *	-o	List of properties to display.  Defaults to
6342  *		"name,size,allocated,free,expandsize,fragmentation,capacity,"
6343  *		"dedupratio,health,altroot"
6344  *	-p	Display values in parsable (exact) format.
6345  *	-P	Display full path for vdev name.
6346  *	-T	Display a timestamp in date(1) or Unix format
6347  *
6348  * List all pools in the system, whether or not they're healthy.  Output space
6349  * statistics for each one, as well as health status summary.
6350  */
6351 int
6352 zpool_do_list(int argc, char **argv)
6353 {
6354 	int c;
6355 	int ret = 0;
6356 	list_cbdata_t cb = { 0 };
6357 	static char default_props[] =
6358 	    "name,size,allocated,free,checkpoint,expandsize,fragmentation,"
6359 	    "capacity,dedupratio,health,altroot";
6360 	char *props = default_props;
6361 	float interval = 0;
6362 	unsigned long count = 0;
6363 	zpool_list_t *list;
6364 	boolean_t first = B_TRUE;
6365 	current_prop_type = ZFS_TYPE_POOL;
6366 
6367 	/* check options */
6368 	while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
6369 		switch (c) {
6370 		case 'g':
6371 			cb.cb_name_flags |= VDEV_NAME_GUID;
6372 			break;
6373 		case 'H':
6374 			cb.cb_scripted = B_TRUE;
6375 			break;
6376 		case 'L':
6377 			cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
6378 			break;
6379 		case 'o':
6380 			props = optarg;
6381 			break;
6382 		case 'P':
6383 			cb.cb_name_flags |= VDEV_NAME_PATH;
6384 			break;
6385 		case 'p':
6386 			cb.cb_literal = B_TRUE;
6387 			break;
6388 		case 'T':
6389 			get_timestamp_arg(*optarg);
6390 			break;
6391 		case 'v':
6392 			cb.cb_verbose = B_TRUE;
6393 			cb.cb_namewidth = 8;	/* 8 until precalc is avail */
6394 			break;
6395 		case ':':
6396 			(void) fprintf(stderr, gettext("missing argument for "
6397 			    "'%c' option\n"), optopt);
6398 			usage(B_FALSE);
6399 			break;
6400 		case '?':
6401 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6402 			    optopt);
6403 			usage(B_FALSE);
6404 		}
6405 	}
6406 
6407 	argc -= optind;
6408 	argv += optind;
6409 
6410 	get_interval_count(&argc, argv, &interval, &count);
6411 
6412 	if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
6413 		usage(B_FALSE);
6414 
6415 	for (;;) {
6416 		if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
6417 		    ZFS_TYPE_POOL, cb.cb_literal, &ret)) == NULL)
6418 			return (1);
6419 
6420 		if (pool_list_count(list) == 0)
6421 			break;
6422 
6423 		cb.cb_namewidth = 0;
6424 		(void) pool_list_iter(list, B_FALSE, get_namewidth_list, &cb);
6425 
6426 		if (timestamp_fmt != NODATE)
6427 			print_timestamp(timestamp_fmt);
6428 
6429 		if (!cb.cb_scripted && (first || cb.cb_verbose)) {
6430 			print_header(&cb);
6431 			first = B_FALSE;
6432 		}
6433 		ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
6434 
6435 		if (interval == 0)
6436 			break;
6437 
6438 		if (count != 0 && --count == 0)
6439 			break;
6440 
6441 		pool_list_free(list);
6442 		(void) fsleep(interval);
6443 	}
6444 
6445 	if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
6446 		(void) printf(gettext("no pools available\n"));
6447 		ret = 0;
6448 	}
6449 
6450 	pool_list_free(list);
6451 	zprop_free_list(cb.cb_proplist);
6452 	return (ret);
6453 }
6454 
6455 static int
6456 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
6457 {
6458 	boolean_t force = B_FALSE;
6459 	boolean_t rebuild = B_FALSE;
6460 	boolean_t wait = B_FALSE;
6461 	int c;
6462 	nvlist_t *nvroot;
6463 	char *poolname, *old_disk, *new_disk;
6464 	zpool_handle_t *zhp;
6465 	nvlist_t *props = NULL;
6466 	char *propval;
6467 	int ret;
6468 
6469 	/* check options */
6470 	while ((c = getopt(argc, argv, "fo:sw")) != -1) {
6471 		switch (c) {
6472 		case 'f':
6473 			force = B_TRUE;
6474 			break;
6475 		case 'o':
6476 			if ((propval = strchr(optarg, '=')) == NULL) {
6477 				(void) fprintf(stderr, gettext("missing "
6478 				    "'=' for -o option\n"));
6479 				usage(B_FALSE);
6480 			}
6481 			*propval = '\0';
6482 			propval++;
6483 
6484 			if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
6485 			    (add_prop_list(optarg, propval, &props, B_TRUE)))
6486 				usage(B_FALSE);
6487 			break;
6488 		case 's':
6489 			rebuild = B_TRUE;
6490 			break;
6491 		case 'w':
6492 			wait = B_TRUE;
6493 			break;
6494 		case '?':
6495 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6496 			    optopt);
6497 			usage(B_FALSE);
6498 		}
6499 	}
6500 
6501 	argc -= optind;
6502 	argv += optind;
6503 
6504 	/* get pool name and check number of arguments */
6505 	if (argc < 1) {
6506 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
6507 		usage(B_FALSE);
6508 	}
6509 
6510 	poolname = argv[0];
6511 
6512 	if (argc < 2) {
6513 		(void) fprintf(stderr,
6514 		    gettext("missing <device> specification\n"));
6515 		usage(B_FALSE);
6516 	}
6517 
6518 	old_disk = argv[1];
6519 
6520 	if (argc < 3) {
6521 		if (!replacing) {
6522 			(void) fprintf(stderr,
6523 			    gettext("missing <new_device> specification\n"));
6524 			usage(B_FALSE);
6525 		}
6526 		new_disk = old_disk;
6527 		argc -= 1;
6528 		argv += 1;
6529 	} else {
6530 		new_disk = argv[2];
6531 		argc -= 2;
6532 		argv += 2;
6533 	}
6534 
6535 	if (argc > 1) {
6536 		(void) fprintf(stderr, gettext("too many arguments\n"));
6537 		usage(B_FALSE);
6538 	}
6539 
6540 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
6541 		nvlist_free(props);
6542 		return (1);
6543 	}
6544 
6545 	if (zpool_get_config(zhp, NULL) == NULL) {
6546 		(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
6547 		    poolname);
6548 		zpool_close(zhp);
6549 		nvlist_free(props);
6550 		return (1);
6551 	}
6552 
6553 	/* unless manually specified use "ashift" pool property (if set) */
6554 	if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
6555 		int intval;
6556 		zprop_source_t src;
6557 		char strval[ZPOOL_MAXPROPLEN];
6558 
6559 		intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
6560 		if (src != ZPROP_SRC_DEFAULT) {
6561 			(void) sprintf(strval, "%" PRId32, intval);
6562 			verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
6563 			    &props, B_TRUE) == 0);
6564 		}
6565 	}
6566 
6567 	nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
6568 	    argc, argv);
6569 	if (nvroot == NULL) {
6570 		zpool_close(zhp);
6571 		nvlist_free(props);
6572 		return (1);
6573 	}
6574 
6575 	ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing,
6576 	    rebuild);
6577 
6578 	if (ret == 0 && wait)
6579 		ret = zpool_wait(zhp,
6580 		    replacing ? ZPOOL_WAIT_REPLACE : ZPOOL_WAIT_RESILVER);
6581 
6582 	nvlist_free(props);
6583 	nvlist_free(nvroot);
6584 	zpool_close(zhp);
6585 
6586 	return (ret);
6587 }
6588 
6589 /*
6590  * zpool replace [-fsw] [-o property=value] <pool> <device> <new_device>
6591  *
6592  *	-f	Force attach, even if <new_device> appears to be in use.
6593  *	-s	Use sequential instead of healing reconstruction for resilver.
6594  *	-o	Set property=value.
6595  *	-w	Wait for replacing to complete before returning
6596  *
6597  * Replace <device> with <new_device>.
6598  */
6599 /* ARGSUSED */
6600 int
6601 zpool_do_replace(int argc, char **argv)
6602 {
6603 	return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
6604 }
6605 
6606 /*
6607  * zpool attach [-fsw] [-o property=value] <pool> <device> <new_device>
6608  *
6609  *	-f	Force attach, even if <new_device> appears to be in use.
6610  *	-s	Use sequential instead of healing reconstruction for resilver.
6611  *	-o	Set property=value.
6612  *	-w	Wait for resilvering to complete before returning
6613  *
6614  * Attach <new_device> to the mirror containing <device>.  If <device> is not
6615  * part of a mirror, then <device> will be transformed into a mirror of
6616  * <device> and <new_device>.  In either case, <new_device> will begin life
6617  * with a DTL of [0, now], and will immediately begin to resilver itself.
6618  */
6619 int
6620 zpool_do_attach(int argc, char **argv)
6621 {
6622 	return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
6623 }
6624 
6625 /*
6626  * zpool detach [-f] <pool> <device>
6627  *
6628  *	-f	Force detach of <device>, even if DTLs argue against it
6629  *		(not supported yet)
6630  *
6631  * Detach a device from a mirror.  The operation will be refused if <device>
6632  * is the last device in the mirror, or if the DTLs indicate that this device
6633  * has the only valid copy of some data.
6634  */
6635 /* ARGSUSED */
6636 int
6637 zpool_do_detach(int argc, char **argv)
6638 {
6639 	int c;
6640 	char *poolname, *path;
6641 	zpool_handle_t *zhp;
6642 	int ret;
6643 
6644 	/* check options */
6645 	while ((c = getopt(argc, argv, "")) != -1) {
6646 		switch (c) {
6647 		case '?':
6648 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6649 			    optopt);
6650 			usage(B_FALSE);
6651 		}
6652 	}
6653 
6654 	argc -= optind;
6655 	argv += optind;
6656 
6657 	/* get pool name and check number of arguments */
6658 	if (argc < 1) {
6659 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
6660 		usage(B_FALSE);
6661 	}
6662 
6663 	if (argc < 2) {
6664 		(void) fprintf(stderr,
6665 		    gettext("missing <device> specification\n"));
6666 		usage(B_FALSE);
6667 	}
6668 
6669 	poolname = argv[0];
6670 	path = argv[1];
6671 
6672 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6673 		return (1);
6674 
6675 	ret = zpool_vdev_detach(zhp, path);
6676 
6677 	zpool_close(zhp);
6678 
6679 	return (ret);
6680 }
6681 
6682 /*
6683  * zpool split [-gLnP] [-o prop=val] ...
6684  *		[-o mntopt] ...
6685  *		[-R altroot] <pool> <newpool> [<device> ...]
6686  *
6687  *	-g      Display guid for individual vdev name.
6688  *	-L	Follow links when resolving vdev path name.
6689  *	-n	Do not split the pool, but display the resulting layout if
6690  *		it were to be split.
6691  *	-o	Set property=value, or set mount options.
6692  *	-P	Display full path for vdev name.
6693  *	-R	Mount the split-off pool under an alternate root.
6694  *	-l	Load encryption keys while importing.
6695  *
6696  * Splits the named pool and gives it the new pool name.  Devices to be split
6697  * off may be listed, provided that no more than one device is specified
6698  * per top-level vdev mirror.  The newly split pool is left in an exported
6699  * state unless -R is specified.
6700  *
6701  * Restrictions: the top-level of the pool pool must only be made up of
6702  * mirrors; all devices in the pool must be healthy; no device may be
6703  * undergoing a resilvering operation.
6704  */
6705 int
6706 zpool_do_split(int argc, char **argv)
6707 {
6708 	char *srcpool, *newpool, *propval;
6709 	char *mntopts = NULL;
6710 	splitflags_t flags;
6711 	int c, ret = 0;
6712 	boolean_t loadkeys = B_FALSE;
6713 	zpool_handle_t *zhp;
6714 	nvlist_t *config, *props = NULL;
6715 
6716 	flags.dryrun = B_FALSE;
6717 	flags.import = B_FALSE;
6718 	flags.name_flags = 0;
6719 
6720 	/* check options */
6721 	while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
6722 		switch (c) {
6723 		case 'g':
6724 			flags.name_flags |= VDEV_NAME_GUID;
6725 			break;
6726 		case 'L':
6727 			flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
6728 			break;
6729 		case 'R':
6730 			flags.import = B_TRUE;
6731 			if (add_prop_list(
6732 			    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
6733 			    &props, B_TRUE) != 0) {
6734 				nvlist_free(props);
6735 				usage(B_FALSE);
6736 			}
6737 			break;
6738 		case 'l':
6739 			loadkeys = B_TRUE;
6740 			break;
6741 		case 'n':
6742 			flags.dryrun = B_TRUE;
6743 			break;
6744 		case 'o':
6745 			if ((propval = strchr(optarg, '=')) != NULL) {
6746 				*propval = '\0';
6747 				propval++;
6748 				if (add_prop_list(optarg, propval,
6749 				    &props, B_TRUE) != 0) {
6750 					nvlist_free(props);
6751 					usage(B_FALSE);
6752 				}
6753 			} else {
6754 				mntopts = optarg;
6755 			}
6756 			break;
6757 		case 'P':
6758 			flags.name_flags |= VDEV_NAME_PATH;
6759 			break;
6760 		case ':':
6761 			(void) fprintf(stderr, gettext("missing argument for "
6762 			    "'%c' option\n"), optopt);
6763 			usage(B_FALSE);
6764 			break;
6765 		case '?':
6766 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6767 			    optopt);
6768 			usage(B_FALSE);
6769 			break;
6770 		}
6771 	}
6772 
6773 	if (!flags.import && mntopts != NULL) {
6774 		(void) fprintf(stderr, gettext("setting mntopts is only "
6775 		    "valid when importing the pool\n"));
6776 		usage(B_FALSE);
6777 	}
6778 
6779 	if (!flags.import && loadkeys) {
6780 		(void) fprintf(stderr, gettext("loading keys is only "
6781 		    "valid when importing the pool\n"));
6782 		usage(B_FALSE);
6783 	}
6784 
6785 	argc -= optind;
6786 	argv += optind;
6787 
6788 	if (argc < 1) {
6789 		(void) fprintf(stderr, gettext("Missing pool name\n"));
6790 		usage(B_FALSE);
6791 	}
6792 	if (argc < 2) {
6793 		(void) fprintf(stderr, gettext("Missing new pool name\n"));
6794 		usage(B_FALSE);
6795 	}
6796 
6797 	srcpool = argv[0];
6798 	newpool = argv[1];
6799 
6800 	argc -= 2;
6801 	argv += 2;
6802 
6803 	if ((zhp = zpool_open(g_zfs, srcpool)) == NULL) {
6804 		nvlist_free(props);
6805 		return (1);
6806 	}
6807 
6808 	config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
6809 	if (config == NULL) {
6810 		ret = 1;
6811 	} else {
6812 		if (flags.dryrun) {
6813 			(void) printf(gettext("would create '%s' with the "
6814 			    "following layout:\n\n"), newpool);
6815 			print_vdev_tree(NULL, newpool, config, 0, "",
6816 			    flags.name_flags);
6817 			print_vdev_tree(NULL, "dedup", config, 0,
6818 			    VDEV_ALLOC_BIAS_DEDUP, 0);
6819 			print_vdev_tree(NULL, "special", config, 0,
6820 			    VDEV_ALLOC_BIAS_SPECIAL, 0);
6821 		}
6822 	}
6823 
6824 	zpool_close(zhp);
6825 
6826 	if (ret != 0 || flags.dryrun || !flags.import) {
6827 		nvlist_free(config);
6828 		nvlist_free(props);
6829 		return (ret);
6830 	}
6831 
6832 	/*
6833 	 * The split was successful. Now we need to open the new
6834 	 * pool and import it.
6835 	 */
6836 	if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL) {
6837 		nvlist_free(config);
6838 		nvlist_free(props);
6839 		return (1);
6840 	}
6841 
6842 	if (loadkeys) {
6843 		ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
6844 		if (ret != 0)
6845 			ret = 1;
6846 	}
6847 
6848 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
6849 	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
6850 		ret = 1;
6851 		(void) fprintf(stderr, gettext("Split was successful, but "
6852 		    "the datasets could not all be mounted\n"));
6853 		(void) fprintf(stderr, gettext("Try doing '%s' with a "
6854 		    "different altroot\n"), "zpool import");
6855 	}
6856 	zpool_close(zhp);
6857 	nvlist_free(config);
6858 	nvlist_free(props);
6859 
6860 	return (ret);
6861 }
6862 
6863 
6864 
6865 /*
6866  * zpool online <pool> <device> ...
6867  */
6868 int
6869 zpool_do_online(int argc, char **argv)
6870 {
6871 	int c, i;
6872 	char *poolname;
6873 	zpool_handle_t *zhp;
6874 	int ret = 0;
6875 	vdev_state_t newstate;
6876 	int flags = 0;
6877 
6878 	/* check options */
6879 	while ((c = getopt(argc, argv, "e")) != -1) {
6880 		switch (c) {
6881 		case 'e':
6882 			flags |= ZFS_ONLINE_EXPAND;
6883 			break;
6884 		case '?':
6885 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6886 			    optopt);
6887 			usage(B_FALSE);
6888 		}
6889 	}
6890 
6891 	argc -= optind;
6892 	argv += optind;
6893 
6894 	/* get pool name and check number of arguments */
6895 	if (argc < 1) {
6896 		(void) fprintf(stderr, gettext("missing pool name\n"));
6897 		usage(B_FALSE);
6898 	}
6899 	if (argc < 2) {
6900 		(void) fprintf(stderr, gettext("missing device name\n"));
6901 		usage(B_FALSE);
6902 	}
6903 
6904 	poolname = argv[0];
6905 
6906 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6907 		return (1);
6908 
6909 	for (i = 1; i < argc; i++) {
6910 		if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
6911 			if (newstate != VDEV_STATE_HEALTHY) {
6912 				(void) printf(gettext("warning: device '%s' "
6913 				    "onlined, but remains in faulted state\n"),
6914 				    argv[i]);
6915 				if (newstate == VDEV_STATE_FAULTED)
6916 					(void) printf(gettext("use 'zpool "
6917 					    "clear' to restore a faulted "
6918 					    "device\n"));
6919 				else
6920 					(void) printf(gettext("use 'zpool "
6921 					    "replace' to replace devices "
6922 					    "that are no longer present\n"));
6923 			}
6924 		} else {
6925 			ret = 1;
6926 		}
6927 	}
6928 
6929 	zpool_close(zhp);
6930 
6931 	return (ret);
6932 }
6933 
6934 /*
6935  * zpool offline [-ft] <pool> <device> ...
6936  *
6937  *	-f	Force the device into a faulted state.
6938  *
6939  *	-t	Only take the device off-line temporarily.  The offline/faulted
6940  *		state will not be persistent across reboots.
6941  */
6942 /* ARGSUSED */
6943 int
6944 zpool_do_offline(int argc, char **argv)
6945 {
6946 	int c, i;
6947 	char *poolname;
6948 	zpool_handle_t *zhp;
6949 	int ret = 0;
6950 	boolean_t istmp = B_FALSE;
6951 	boolean_t fault = B_FALSE;
6952 
6953 	/* check options */
6954 	while ((c = getopt(argc, argv, "ft")) != -1) {
6955 		switch (c) {
6956 		case 'f':
6957 			fault = B_TRUE;
6958 			break;
6959 		case 't':
6960 			istmp = B_TRUE;
6961 			break;
6962 		case '?':
6963 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6964 			    optopt);
6965 			usage(B_FALSE);
6966 		}
6967 	}
6968 
6969 	argc -= optind;
6970 	argv += optind;
6971 
6972 	/* get pool name and check number of arguments */
6973 	if (argc < 1) {
6974 		(void) fprintf(stderr, gettext("missing pool name\n"));
6975 		usage(B_FALSE);
6976 	}
6977 	if (argc < 2) {
6978 		(void) fprintf(stderr, gettext("missing device name\n"));
6979 		usage(B_FALSE);
6980 	}
6981 
6982 	poolname = argv[0];
6983 
6984 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6985 		return (1);
6986 
6987 	for (i = 1; i < argc; i++) {
6988 		if (fault) {
6989 			uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
6990 			vdev_aux_t aux;
6991 			if (istmp == B_FALSE) {
6992 				/* Force the fault to persist across imports */
6993 				aux = VDEV_AUX_EXTERNAL_PERSIST;
6994 			} else {
6995 				aux = VDEV_AUX_EXTERNAL;
6996 			}
6997 
6998 			if (guid == 0 || zpool_vdev_fault(zhp, guid, aux) != 0)
6999 				ret = 1;
7000 		} else {
7001 			if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
7002 				ret = 1;
7003 		}
7004 	}
7005 
7006 	zpool_close(zhp);
7007 
7008 	return (ret);
7009 }
7010 
7011 /*
7012  * zpool clear <pool> [device]
7013  *
7014  * Clear all errors associated with a pool or a particular device.
7015  */
7016 int
7017 zpool_do_clear(int argc, char **argv)
7018 {
7019 	int c;
7020 	int ret = 0;
7021 	boolean_t dryrun = B_FALSE;
7022 	boolean_t do_rewind = B_FALSE;
7023 	boolean_t xtreme_rewind = B_FALSE;
7024 	uint32_t rewind_policy = ZPOOL_NO_REWIND;
7025 	nvlist_t *policy = NULL;
7026 	zpool_handle_t *zhp;
7027 	char *pool, *device;
7028 
7029 	/* check options */
7030 	while ((c = getopt(argc, argv, "FnX")) != -1) {
7031 		switch (c) {
7032 		case 'F':
7033 			do_rewind = B_TRUE;
7034 			break;
7035 		case 'n':
7036 			dryrun = B_TRUE;
7037 			break;
7038 		case 'X':
7039 			xtreme_rewind = B_TRUE;
7040 			break;
7041 		case '?':
7042 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
7043 			    optopt);
7044 			usage(B_FALSE);
7045 		}
7046 	}
7047 
7048 	argc -= optind;
7049 	argv += optind;
7050 
7051 	if (argc < 1) {
7052 		(void) fprintf(stderr, gettext("missing pool name\n"));
7053 		usage(B_FALSE);
7054 	}
7055 
7056 	if (argc > 2) {
7057 		(void) fprintf(stderr, gettext("too many arguments\n"));
7058 		usage(B_FALSE);
7059 	}
7060 
7061 	if ((dryrun || xtreme_rewind) && !do_rewind) {
7062 		(void) fprintf(stderr,
7063 		    gettext("-n or -X only meaningful with -F\n"));
7064 		usage(B_FALSE);
7065 	}
7066 	if (dryrun)
7067 		rewind_policy = ZPOOL_TRY_REWIND;
7068 	else if (do_rewind)
7069 		rewind_policy = ZPOOL_DO_REWIND;
7070 	if (xtreme_rewind)
7071 		rewind_policy |= ZPOOL_EXTREME_REWIND;
7072 
7073 	/* In future, further rewind policy choices can be passed along here */
7074 	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
7075 	    nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
7076 	    rewind_policy) != 0) {
7077 		return (1);
7078 	}
7079 
7080 	pool = argv[0];
7081 	device = argc == 2 ? argv[1] : NULL;
7082 
7083 	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
7084 		nvlist_free(policy);
7085 		return (1);
7086 	}
7087 
7088 	if (zpool_clear(zhp, device, policy) != 0)
7089 		ret = 1;
7090 
7091 	zpool_close(zhp);
7092 
7093 	nvlist_free(policy);
7094 
7095 	return (ret);
7096 }
7097 
7098 /*
7099  * zpool reguid <pool>
7100  */
7101 int
7102 zpool_do_reguid(int argc, char **argv)
7103 {
7104 	int c;
7105 	char *poolname;
7106 	zpool_handle_t *zhp;
7107 	int ret = 0;
7108 
7109 	/* check options */
7110 	while ((c = getopt(argc, argv, "")) != -1) {
7111 		switch (c) {
7112 		case '?':
7113 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
7114 			    optopt);
7115 			usage(B_FALSE);
7116 		}
7117 	}
7118 
7119 	argc -= optind;
7120 	argv += optind;
7121 
7122 	/* get pool name and check number of arguments */
7123 	if (argc < 1) {
7124 		(void) fprintf(stderr, gettext("missing pool name\n"));
7125 		usage(B_FALSE);
7126 	}
7127 
7128 	if (argc > 1) {
7129 		(void) fprintf(stderr, gettext("too many arguments\n"));
7130 		usage(B_FALSE);
7131 	}
7132 
7133 	poolname = argv[0];
7134 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
7135 		return (1);
7136 
7137 	ret = zpool_reguid(zhp);
7138 
7139 	zpool_close(zhp);
7140 	return (ret);
7141 }
7142 
7143 
7144 /*
7145  * zpool reopen <pool>
7146  *
7147  * Reopen the pool so that the kernel can update the sizes of all vdevs.
7148  */
7149 int
7150 zpool_do_reopen(int argc, char **argv)
7151 {
7152 	int c;
7153 	int ret = 0;
7154 	boolean_t scrub_restart = B_TRUE;
7155 
7156 	/* check options */
7157 	while ((c = getopt(argc, argv, "n")) != -1) {
7158 		switch (c) {
7159 		case 'n':
7160 			scrub_restart = B_FALSE;
7161 			break;
7162 		case '?':
7163 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
7164 			    optopt);
7165 			usage(B_FALSE);
7166 		}
7167 	}
7168 
7169 	argc -= optind;
7170 	argv += optind;
7171 
7172 	/* if argc == 0 we will execute zpool_reopen_one on all pools */
7173 	ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7174 	    B_FALSE, zpool_reopen_one, &scrub_restart);
7175 
7176 	return (ret);
7177 }
7178 
7179 typedef struct scrub_cbdata {
7180 	int	cb_type;
7181 	pool_scrub_cmd_t cb_scrub_cmd;
7182 } scrub_cbdata_t;
7183 
7184 static boolean_t
7185 zpool_has_checkpoint(zpool_handle_t *zhp)
7186 {
7187 	nvlist_t *config, *nvroot;
7188 
7189 	config = zpool_get_config(zhp, NULL);
7190 
7191 	if (config != NULL) {
7192 		pool_checkpoint_stat_t *pcs = NULL;
7193 		uint_t c;
7194 
7195 		nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
7196 		(void) nvlist_lookup_uint64_array(nvroot,
7197 		    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
7198 
7199 		if (pcs == NULL || pcs->pcs_state == CS_NONE)
7200 			return (B_FALSE);
7201 
7202 		assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS ||
7203 		    pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
7204 		return (B_TRUE);
7205 	}
7206 
7207 	return (B_FALSE);
7208 }
7209 
7210 static int
7211 scrub_callback(zpool_handle_t *zhp, void *data)
7212 {
7213 	scrub_cbdata_t *cb = data;
7214 	int err;
7215 
7216 	/*
7217 	 * Ignore faulted pools.
7218 	 */
7219 	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
7220 		(void) fprintf(stderr, gettext("cannot scan '%s': pool is "
7221 		    "currently unavailable\n"), zpool_get_name(zhp));
7222 		return (1);
7223 	}
7224 
7225 	err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
7226 
7227 	if (err == 0 && zpool_has_checkpoint(zhp) &&
7228 	    cb->cb_type == POOL_SCAN_SCRUB) {
7229 		(void) printf(gettext("warning: will not scrub state that "
7230 		    "belongs to the checkpoint of pool '%s'\n"),
7231 		    zpool_get_name(zhp));
7232 	}
7233 
7234 	return (err != 0);
7235 }
7236 
7237 static int
7238 wait_callback(zpool_handle_t *zhp, void *data)
7239 {
7240 	zpool_wait_activity_t *act = data;
7241 	return (zpool_wait(zhp, *act));
7242 }
7243 
7244 /*
7245  * zpool scrub [-s | -p] [-w] <pool> ...
7246  *
7247  *	-s	Stop.  Stops any in-progress scrub.
7248  *	-p	Pause. Pause in-progress scrub.
7249  *	-w	Wait.  Blocks until scrub has completed.
7250  */
7251 int
7252 zpool_do_scrub(int argc, char **argv)
7253 {
7254 	int c;
7255 	scrub_cbdata_t cb;
7256 	boolean_t wait = B_FALSE;
7257 	int error;
7258 
7259 	cb.cb_type = POOL_SCAN_SCRUB;
7260 	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7261 
7262 	/* check options */
7263 	while ((c = getopt(argc, argv, "spw")) != -1) {
7264 		switch (c) {
7265 		case 's':
7266 			cb.cb_type = POOL_SCAN_NONE;
7267 			break;
7268 		case 'p':
7269 			cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
7270 			break;
7271 		case 'w':
7272 			wait = B_TRUE;
7273 			break;
7274 		case '?':
7275 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
7276 			    optopt);
7277 			usage(B_FALSE);
7278 		}
7279 	}
7280 
7281 	if (cb.cb_type == POOL_SCAN_NONE &&
7282 	    cb.cb_scrub_cmd == POOL_SCRUB_PAUSE) {
7283 		(void) fprintf(stderr, gettext("invalid option combination: "
7284 		    "-s and -p are mutually exclusive\n"));
7285 		usage(B_FALSE);
7286 	}
7287 
7288 	if (wait && (cb.cb_type == POOL_SCAN_NONE ||
7289 	    cb.cb_scrub_cmd == POOL_SCRUB_PAUSE)) {
7290 		(void) fprintf(stderr, gettext("invalid option combination: "
7291 		    "-w cannot be used with -p or -s\n"));
7292 		usage(B_FALSE);
7293 	}
7294 
7295 	argc -= optind;
7296 	argv += optind;
7297 
7298 	if (argc < 1) {
7299 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
7300 		usage(B_FALSE);
7301 	}
7302 
7303 	error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7304 	    B_FALSE, scrub_callback, &cb);
7305 
7306 	if (wait && !error) {
7307 		zpool_wait_activity_t act = ZPOOL_WAIT_SCRUB;
7308 		error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7309 		    B_FALSE, wait_callback, &act);
7310 	}
7311 
7312 	return (error);
7313 }
7314 
7315 /*
7316  * zpool resilver <pool> ...
7317  *
7318  *	Restarts any in-progress resilver
7319  */
7320 int
7321 zpool_do_resilver(int argc, char **argv)
7322 {
7323 	int c;
7324 	scrub_cbdata_t cb;
7325 
7326 	cb.cb_type = POOL_SCAN_RESILVER;
7327 	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7328 
7329 	/* check options */
7330 	while ((c = getopt(argc, argv, "")) != -1) {
7331 		switch (c) {
7332 		case '?':
7333 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
7334 			    optopt);
7335 			usage(B_FALSE);
7336 		}
7337 	}
7338 
7339 	argc -= optind;
7340 	argv += optind;
7341 
7342 	if (argc < 1) {
7343 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
7344 		usage(B_FALSE);
7345 	}
7346 
7347 	return (for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7348 	    B_FALSE, scrub_callback, &cb));
7349 }
7350 
7351 /*
7352  * zpool trim [-d] [-r <rate>] [-c | -s] <pool> [<device> ...]
7353  *
7354  *	-c		Cancel. Ends any in-progress trim.
7355  *	-d		Secure trim.  Requires kernel and device support.
7356  *	-r <rate>	Sets the TRIM rate in bytes (per second). Supports
7357  *			adding a multiplier suffix such as 'k' or 'm'.
7358  *	-s		Suspend. TRIM can then be restarted with no flags.
7359  *	-w		Wait. Blocks until trimming has completed.
7360  */
7361 int
7362 zpool_do_trim(int argc, char **argv)
7363 {
7364 	struct option long_options[] = {
7365 		{"cancel",	no_argument,		NULL,	'c'},
7366 		{"secure",	no_argument,		NULL,	'd'},
7367 		{"rate",	required_argument,	NULL,	'r'},
7368 		{"suspend",	no_argument,		NULL,	's'},
7369 		{"wait",	no_argument,		NULL,	'w'},
7370 		{0, 0, 0, 0}
7371 	};
7372 
7373 	pool_trim_func_t cmd_type = POOL_TRIM_START;
7374 	uint64_t rate = 0;
7375 	boolean_t secure = B_FALSE;
7376 	boolean_t wait = B_FALSE;
7377 
7378 	int c;
7379 	while ((c = getopt_long(argc, argv, "cdr:sw", long_options, NULL))
7380 	    != -1) {
7381 		switch (c) {
7382 		case 'c':
7383 			if (cmd_type != POOL_TRIM_START &&
7384 			    cmd_type != POOL_TRIM_CANCEL) {
7385 				(void) fprintf(stderr, gettext("-c cannot be "
7386 				    "combined with other options\n"));
7387 				usage(B_FALSE);
7388 			}
7389 			cmd_type = POOL_TRIM_CANCEL;
7390 			break;
7391 		case 'd':
7392 			if (cmd_type != POOL_TRIM_START) {
7393 				(void) fprintf(stderr, gettext("-d cannot be "
7394 				    "combined with the -c or -s options\n"));
7395 				usage(B_FALSE);
7396 			}
7397 			secure = B_TRUE;
7398 			break;
7399 		case 'r':
7400 			if (cmd_type != POOL_TRIM_START) {
7401 				(void) fprintf(stderr, gettext("-r cannot be "
7402 				    "combined with the -c or -s options\n"));
7403 				usage(B_FALSE);
7404 			}
7405 			if (zfs_nicestrtonum(g_zfs, optarg, &rate) == -1) {
7406 				(void) fprintf(stderr, "%s: %s\n",
7407 				    gettext("invalid value for rate"),
7408 				    libzfs_error_description(g_zfs));
7409 				usage(B_FALSE);
7410 			}
7411 			break;
7412 		case 's':
7413 			if (cmd_type != POOL_TRIM_START &&
7414 			    cmd_type != POOL_TRIM_SUSPEND) {
7415 				(void) fprintf(stderr, gettext("-s cannot be "
7416 				    "combined with other options\n"));
7417 				usage(B_FALSE);
7418 			}
7419 			cmd_type = POOL_TRIM_SUSPEND;
7420 			break;
7421 		case 'w':
7422 			wait = B_TRUE;
7423 			break;
7424 		case '?':
7425 			if (optopt != 0) {
7426 				(void) fprintf(stderr,
7427 				    gettext("invalid option '%c'\n"), optopt);
7428 			} else {
7429 				(void) fprintf(stderr,
7430 				    gettext("invalid option '%s'\n"),
7431 				    argv[optind - 1]);
7432 			}
7433 			usage(B_FALSE);
7434 		}
7435 	}
7436 
7437 	argc -= optind;
7438 	argv += optind;
7439 
7440 	if (argc < 1) {
7441 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
7442 		usage(B_FALSE);
7443 		return (-1);
7444 	}
7445 
7446 	if (wait && (cmd_type != POOL_TRIM_START)) {
7447 		(void) fprintf(stderr, gettext("-w cannot be used with -c or "
7448 		    "-s\n"));
7449 		usage(B_FALSE);
7450 	}
7451 
7452 	char *poolname = argv[0];
7453 	zpool_handle_t *zhp = zpool_open(g_zfs, poolname);
7454 	if (zhp == NULL)
7455 		return (-1);
7456 
7457 	trimflags_t trim_flags = {
7458 		.secure = secure,
7459 		.rate = rate,
7460 		.wait = wait,
7461 	};
7462 
7463 	nvlist_t *vdevs = fnvlist_alloc();
7464 	if (argc == 1) {
7465 		/* no individual leaf vdevs specified, so add them all */
7466 		nvlist_t *config = zpool_get_config(zhp, NULL);
7467 		nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
7468 		    ZPOOL_CONFIG_VDEV_TREE);
7469 		zpool_collect_leaves(zhp, nvroot, vdevs);
7470 		trim_flags.fullpool = B_TRUE;
7471 	} else {
7472 		trim_flags.fullpool = B_FALSE;
7473 		for (int i = 1; i < argc; i++) {
7474 			fnvlist_add_boolean(vdevs, argv[i]);
7475 		}
7476 	}
7477 
7478 	int error = zpool_trim(zhp, cmd_type, vdevs, &trim_flags);
7479 
7480 	fnvlist_free(vdevs);
7481 	zpool_close(zhp);
7482 
7483 	return (error);
7484 }
7485 
7486 /*
7487  * Converts a total number of seconds to a human readable string broken
7488  * down in to days/hours/minutes/seconds.
7489  */
7490 static void
7491 secs_to_dhms(uint64_t total, char *buf)
7492 {
7493 	uint64_t days = total / 60 / 60 / 24;
7494 	uint64_t hours = (total / 60 / 60) % 24;
7495 	uint64_t mins = (total / 60) % 60;
7496 	uint64_t secs = (total % 60);
7497 
7498 	if (days > 0) {
7499 		(void) sprintf(buf, "%llu days %02llu:%02llu:%02llu",
7500 		    (u_longlong_t)days, (u_longlong_t)hours,
7501 		    (u_longlong_t)mins, (u_longlong_t)secs);
7502 	} else {
7503 		(void) sprintf(buf, "%02llu:%02llu:%02llu",
7504 		    (u_longlong_t)hours, (u_longlong_t)mins,
7505 		    (u_longlong_t)secs);
7506 	}
7507 }
7508 
7509 /*
7510  * Print out detailed scrub status.
7511  */
7512 static void
7513 print_scan_scrub_resilver_status(pool_scan_stat_t *ps)
7514 {
7515 	time_t start, end, pause;
7516 	uint64_t pass_scanned, scanned, pass_issued, issued, total;
7517 	uint64_t elapsed, scan_rate, issue_rate;
7518 	double fraction_done;
7519 	char processed_buf[7], scanned_buf[7], issued_buf[7], total_buf[7];
7520 	char srate_buf[7], irate_buf[7], time_buf[32];
7521 
7522 	printf("  ");
7523 	printf_color(ANSI_BOLD, gettext("scan:"));
7524 	printf(" ");
7525 
7526 	/* If there's never been a scan, there's not much to say. */
7527 	if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
7528 	    ps->pss_func >= POOL_SCAN_FUNCS) {
7529 		(void) printf(gettext("none requested\n"));
7530 		return;
7531 	}
7532 
7533 	start = ps->pss_start_time;
7534 	end = ps->pss_end_time;
7535 	pause = ps->pss_pass_scrub_pause;
7536 
7537 	zfs_nicebytes(ps->pss_processed, processed_buf, sizeof (processed_buf));
7538 
7539 	assert(ps->pss_func == POOL_SCAN_SCRUB ||
7540 	    ps->pss_func == POOL_SCAN_RESILVER);
7541 
7542 	/* Scan is finished or canceled. */
7543 	if (ps->pss_state == DSS_FINISHED) {
7544 		secs_to_dhms(end - start, time_buf);
7545 
7546 		if (ps->pss_func == POOL_SCAN_SCRUB) {
7547 			(void) printf(gettext("scrub repaired %s "
7548 			    "in %s with %llu errors on %s"), processed_buf,
7549 			    time_buf, (u_longlong_t)ps->pss_errors,
7550 			    ctime(&end));
7551 		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
7552 			(void) printf(gettext("resilvered %s "
7553 			    "in %s with %llu errors on %s"), processed_buf,
7554 			    time_buf, (u_longlong_t)ps->pss_errors,
7555 			    ctime(&end));
7556 		}
7557 		return;
7558 	} else if (ps->pss_state == DSS_CANCELED) {
7559 		if (ps->pss_func == POOL_SCAN_SCRUB) {
7560 			(void) printf(gettext("scrub canceled on %s"),
7561 			    ctime(&end));
7562 		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
7563 			(void) printf(gettext("resilver canceled on %s"),
7564 			    ctime(&end));
7565 		}
7566 		return;
7567 	}
7568 
7569 	assert(ps->pss_state == DSS_SCANNING);
7570 
7571 	/* Scan is in progress. Resilvers can't be paused. */
7572 	if (ps->pss_func == POOL_SCAN_SCRUB) {
7573 		if (pause == 0) {
7574 			(void) printf(gettext("scrub in progress since %s"),
7575 			    ctime(&start));
7576 		} else {
7577 			(void) printf(gettext("scrub paused since %s"),
7578 			    ctime(&pause));
7579 			(void) printf(gettext("\tscrub started on %s"),
7580 			    ctime(&start));
7581 		}
7582 	} else if (ps->pss_func == POOL_SCAN_RESILVER) {
7583 		(void) printf(gettext("resilver in progress since %s"),
7584 		    ctime(&start));
7585 	}
7586 
7587 	scanned = ps->pss_examined;
7588 	pass_scanned = ps->pss_pass_exam;
7589 	issued = ps->pss_issued;
7590 	pass_issued = ps->pss_pass_issued;
7591 	total = ps->pss_to_examine;
7592 
7593 	/* we are only done with a block once we have issued the IO for it */
7594 	fraction_done = (double)issued / total;
7595 
7596 	/* elapsed time for this pass, rounding up to 1 if it's 0 */
7597 	elapsed = time(NULL) - ps->pss_pass_start;
7598 	elapsed -= ps->pss_pass_scrub_spent_paused;
7599 	elapsed = (elapsed != 0) ? elapsed : 1;
7600 
7601 	scan_rate = pass_scanned / elapsed;
7602 	issue_rate = pass_issued / elapsed;
7603 	uint64_t total_secs_left = (issue_rate != 0 && total >= issued) ?
7604 	    ((total - issued) / issue_rate) : UINT64_MAX;
7605 	secs_to_dhms(total_secs_left, time_buf);
7606 
7607 	/* format all of the numbers we will be reporting */
7608 	zfs_nicebytes(scanned, scanned_buf, sizeof (scanned_buf));
7609 	zfs_nicebytes(issued, issued_buf, sizeof (issued_buf));
7610 	zfs_nicebytes(total, total_buf, sizeof (total_buf));
7611 	zfs_nicebytes(scan_rate, srate_buf, sizeof (srate_buf));
7612 	zfs_nicebytes(issue_rate, irate_buf, sizeof (irate_buf));
7613 
7614 	/* do not print estimated time if we have a paused scrub */
7615 	if (pause == 0) {
7616 		(void) printf(gettext("\t%s scanned at %s/s, "
7617 		    "%s issued at %s/s, %s total\n"),
7618 		    scanned_buf, srate_buf, issued_buf, irate_buf, total_buf);
7619 	} else {
7620 		(void) printf(gettext("\t%s scanned, %s issued, %s total\n"),
7621 		    scanned_buf, issued_buf, total_buf);
7622 	}
7623 
7624 	if (ps->pss_func == POOL_SCAN_RESILVER) {
7625 		(void) printf(gettext("\t%s resilvered, %.2f%% done"),
7626 		    processed_buf, 100 * fraction_done);
7627 	} else if (ps->pss_func == POOL_SCAN_SCRUB) {
7628 		(void) printf(gettext("\t%s repaired, %.2f%% done"),
7629 		    processed_buf, 100 * fraction_done);
7630 	}
7631 
7632 	if (pause == 0) {
7633 		if (total_secs_left != UINT64_MAX &&
7634 		    issue_rate >= 10 * 1024 * 1024) {
7635 			(void) printf(gettext(", %s to go\n"), time_buf);
7636 		} else {
7637 			(void) printf(gettext(", no estimated "
7638 			    "completion time\n"));
7639 		}
7640 	} else {
7641 		(void) printf(gettext("\n"));
7642 	}
7643 }
7644 
7645 static void
7646 print_rebuild_status_impl(vdev_rebuild_stat_t *vrs, char *vdev_name)
7647 {
7648 	if (vrs == NULL || vrs->vrs_state == VDEV_REBUILD_NONE)
7649 		return;
7650 
7651 	printf("  ");
7652 	printf_color(ANSI_BOLD, gettext("scan:"));
7653 	printf(" ");
7654 
7655 	uint64_t bytes_scanned = vrs->vrs_bytes_scanned;
7656 	uint64_t bytes_issued = vrs->vrs_bytes_issued;
7657 	uint64_t bytes_rebuilt = vrs->vrs_bytes_rebuilt;
7658 	uint64_t bytes_est = vrs->vrs_bytes_est;
7659 	uint64_t scan_rate = (vrs->vrs_pass_bytes_scanned /
7660 	    (vrs->vrs_pass_time_ms + 1)) * 1000;
7661 	uint64_t issue_rate = (vrs->vrs_pass_bytes_issued /
7662 	    (vrs->vrs_pass_time_ms + 1)) * 1000;
7663 	double scan_pct = MIN((double)bytes_scanned * 100 /
7664 	    (bytes_est + 1), 100);
7665 
7666 	/* Format all of the numbers we will be reporting */
7667 	char bytes_scanned_buf[7], bytes_issued_buf[7];
7668 	char bytes_rebuilt_buf[7], bytes_est_buf[7];
7669 	char scan_rate_buf[7], issue_rate_buf[7], time_buf[32];
7670 	zfs_nicebytes(bytes_scanned, bytes_scanned_buf,
7671 	    sizeof (bytes_scanned_buf));
7672 	zfs_nicebytes(bytes_issued, bytes_issued_buf,
7673 	    sizeof (bytes_issued_buf));
7674 	zfs_nicebytes(bytes_rebuilt, bytes_rebuilt_buf,
7675 	    sizeof (bytes_rebuilt_buf));
7676 	zfs_nicebytes(bytes_est, bytes_est_buf, sizeof (bytes_est_buf));
7677 	zfs_nicebytes(scan_rate, scan_rate_buf, sizeof (scan_rate_buf));
7678 	zfs_nicebytes(issue_rate, issue_rate_buf, sizeof (issue_rate_buf));
7679 
7680 	time_t start = vrs->vrs_start_time;
7681 	time_t end = vrs->vrs_end_time;
7682 
7683 	/* Rebuild is finished or canceled. */
7684 	if (vrs->vrs_state == VDEV_REBUILD_COMPLETE) {
7685 		secs_to_dhms(vrs->vrs_scan_time_ms / 1000, time_buf);
7686 		(void) printf(gettext("resilvered (%s) %s in %s "
7687 		    "with %llu errors on %s"), vdev_name, bytes_rebuilt_buf,
7688 		    time_buf, (u_longlong_t)vrs->vrs_errors, ctime(&end));
7689 		return;
7690 	} else if (vrs->vrs_state == VDEV_REBUILD_CANCELED) {
7691 		(void) printf(gettext("resilver (%s) canceled on %s"),
7692 		    vdev_name, ctime(&end));
7693 		return;
7694 	} else if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7695 		(void) printf(gettext("resilver (%s) in progress since %s"),
7696 		    vdev_name, ctime(&start));
7697 	}
7698 
7699 	assert(vrs->vrs_state == VDEV_REBUILD_ACTIVE);
7700 
7701 	secs_to_dhms(MAX((int64_t)bytes_est - (int64_t)bytes_scanned, 0) /
7702 	    MAX(scan_rate, 1), time_buf);
7703 
7704 	(void) printf(gettext("\t%s scanned at %s/s, %s issued %s/s, "
7705 	    "%s total\n"), bytes_scanned_buf, scan_rate_buf,
7706 	    bytes_issued_buf, issue_rate_buf, bytes_est_buf);
7707 	(void) printf(gettext("\t%s resilvered, %.2f%% done"),
7708 	    bytes_rebuilt_buf, scan_pct);
7709 
7710 	if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7711 		if (scan_rate >= 10 * 1024 * 1024) {
7712 			(void) printf(gettext(", %s to go\n"), time_buf);
7713 		} else {
7714 			(void) printf(gettext(", no estimated "
7715 			    "completion time\n"));
7716 		}
7717 	} else {
7718 		(void) printf(gettext("\n"));
7719 	}
7720 }
7721 
7722 /*
7723  * Print rebuild status for top-level vdevs.
7724  */
7725 static void
7726 print_rebuild_status(zpool_handle_t *zhp, nvlist_t *nvroot)
7727 {
7728 	nvlist_t **child;
7729 	uint_t children;
7730 
7731 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7732 	    &child, &children) != 0)
7733 		children = 0;
7734 
7735 	for (uint_t c = 0; c < children; c++) {
7736 		vdev_rebuild_stat_t *vrs;
7737 		uint_t i;
7738 
7739 		if (nvlist_lookup_uint64_array(child[c],
7740 		    ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
7741 			char *name = zpool_vdev_name(g_zfs, zhp,
7742 			    child[c], VDEV_NAME_TYPE_ID);
7743 			print_rebuild_status_impl(vrs, name);
7744 			free(name);
7745 		}
7746 	}
7747 }
7748 
7749 /*
7750  * As we don't scrub checkpointed blocks, we want to warn the user that we
7751  * skipped scanning some blocks if a checkpoint exists or existed at any
7752  * time during the scan.  If a sequential instead of healing reconstruction
7753  * was performed then the blocks were reconstructed.  However, their checksums
7754  * have not been verified so we still print the warning.
7755  */
7756 static void
7757 print_checkpoint_scan_warning(pool_scan_stat_t *ps, pool_checkpoint_stat_t *pcs)
7758 {
7759 	if (ps == NULL || pcs == NULL)
7760 		return;
7761 
7762 	if (pcs->pcs_state == CS_NONE ||
7763 	    pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
7764 		return;
7765 
7766 	assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);
7767 
7768 	if (ps->pss_state == DSS_NONE)
7769 		return;
7770 
7771 	if ((ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) &&
7772 	    ps->pss_end_time < pcs->pcs_start_time)
7773 		return;
7774 
7775 	if (ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) {
7776 		(void) printf(gettext("    scan warning: skipped blocks "
7777 		    "that are only referenced by the checkpoint.\n"));
7778 	} else {
7779 		assert(ps->pss_state == DSS_SCANNING);
7780 		(void) printf(gettext("    scan warning: skipping blocks "
7781 		    "that are only referenced by the checkpoint.\n"));
7782 	}
7783 }
7784 
7785 /*
7786  * Returns B_TRUE if there is an active rebuild in progress.  Otherwise,
7787  * B_FALSE is returned and 'rebuild_end_time' is set to the end time for
7788  * the last completed (or cancelled) rebuild.
7789  */
7790 static boolean_t
7791 check_rebuilding(nvlist_t *nvroot, uint64_t *rebuild_end_time)
7792 {
7793 	nvlist_t **child;
7794 	uint_t children;
7795 	boolean_t rebuilding = B_FALSE;
7796 	uint64_t end_time = 0;
7797 
7798 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7799 	    &child, &children) != 0)
7800 		children = 0;
7801 
7802 	for (uint_t c = 0; c < children; c++) {
7803 		vdev_rebuild_stat_t *vrs;
7804 		uint_t i;
7805 
7806 		if (nvlist_lookup_uint64_array(child[c],
7807 		    ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
7808 
7809 			if (vrs->vrs_end_time > end_time)
7810 				end_time = vrs->vrs_end_time;
7811 
7812 			if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7813 				rebuilding = B_TRUE;
7814 				end_time = 0;
7815 				break;
7816 			}
7817 		}
7818 	}
7819 
7820 	if (rebuild_end_time != NULL)
7821 		*rebuild_end_time = end_time;
7822 
7823 	return (rebuilding);
7824 }
7825 
7826 /*
7827  * Print the scan status.
7828  */
7829 static void
7830 print_scan_status(zpool_handle_t *zhp, nvlist_t *nvroot)
7831 {
7832 	uint64_t rebuild_end_time = 0, resilver_end_time = 0;
7833 	boolean_t have_resilver = B_FALSE, have_scrub = B_FALSE;
7834 	boolean_t active_resilver = B_FALSE;
7835 	pool_checkpoint_stat_t *pcs = NULL;
7836 	pool_scan_stat_t *ps = NULL;
7837 	uint_t c;
7838 
7839 	if (nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
7840 	    (uint64_t **)&ps, &c) == 0) {
7841 		if (ps->pss_func == POOL_SCAN_RESILVER) {
7842 			resilver_end_time = ps->pss_end_time;
7843 			active_resilver = (ps->pss_state == DSS_SCANNING);
7844 		}
7845 
7846 		have_resilver = (ps->pss_func == POOL_SCAN_RESILVER);
7847 		have_scrub = (ps->pss_func == POOL_SCAN_SCRUB);
7848 	}
7849 
7850 	boolean_t active_rebuild = check_rebuilding(nvroot, &rebuild_end_time);
7851 	boolean_t have_rebuild = (active_rebuild || (rebuild_end_time > 0));
7852 
7853 	/* Always print the scrub status when available. */
7854 	if (have_scrub)
7855 		print_scan_scrub_resilver_status(ps);
7856 
7857 	/*
7858 	 * When there is an active resilver or rebuild print its status.
7859 	 * Otherwise print the status of the last resilver or rebuild.
7860 	 */
7861 	if (active_resilver || (!active_rebuild && have_resilver &&
7862 	    resilver_end_time && resilver_end_time > rebuild_end_time)) {
7863 		print_scan_scrub_resilver_status(ps);
7864 	} else if (active_rebuild || (!active_resilver && have_rebuild &&
7865 	    rebuild_end_time && rebuild_end_time > resilver_end_time)) {
7866 		print_rebuild_status(zhp, nvroot);
7867 	}
7868 
7869 	(void) nvlist_lookup_uint64_array(nvroot,
7870 	    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
7871 	print_checkpoint_scan_warning(ps, pcs);
7872 }
7873 
7874 /*
7875  * Print out detailed removal status.
7876  */
7877 static void
7878 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
7879 {
7880 	char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
7881 	time_t start, end;
7882 	nvlist_t *config, *nvroot;
7883 	nvlist_t **child;
7884 	uint_t children;
7885 	char *vdev_name;
7886 
7887 	if (prs == NULL || prs->prs_state == DSS_NONE)
7888 		return;
7889 
7890 	/*
7891 	 * Determine name of vdev.
7892 	 */
7893 	config = zpool_get_config(zhp, NULL);
7894 	nvroot = fnvlist_lookup_nvlist(config,
7895 	    ZPOOL_CONFIG_VDEV_TREE);
7896 	verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7897 	    &child, &children) == 0);
7898 	assert(prs->prs_removing_vdev < children);
7899 	vdev_name = zpool_vdev_name(g_zfs, zhp,
7900 	    child[prs->prs_removing_vdev], B_TRUE);
7901 
7902 	printf_color(ANSI_BOLD, gettext("remove: "));
7903 
7904 	start = prs->prs_start_time;
7905 	end = prs->prs_end_time;
7906 	zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
7907 
7908 	/*
7909 	 * Removal is finished or canceled.
7910 	 */
7911 	if (prs->prs_state == DSS_FINISHED) {
7912 		uint64_t minutes_taken = (end - start) / 60;
7913 
7914 		(void) printf(gettext("Removal of vdev %llu copied %s "
7915 		    "in %lluh%um, completed on %s"),
7916 		    (longlong_t)prs->prs_removing_vdev,
7917 		    copied_buf,
7918 		    (u_longlong_t)(minutes_taken / 60),
7919 		    (uint_t)(minutes_taken % 60),
7920 		    ctime((time_t *)&end));
7921 	} else if (prs->prs_state == DSS_CANCELED) {
7922 		(void) printf(gettext("Removal of %s canceled on %s"),
7923 		    vdev_name, ctime(&end));
7924 	} else {
7925 		uint64_t copied, total, elapsed, mins_left, hours_left;
7926 		double fraction_done;
7927 		uint_t rate;
7928 
7929 		assert(prs->prs_state == DSS_SCANNING);
7930 
7931 		/*
7932 		 * Removal is in progress.
7933 		 */
7934 		(void) printf(gettext(
7935 		    "Evacuation of %s in progress since %s"),
7936 		    vdev_name, ctime(&start));
7937 
7938 		copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
7939 		total = prs->prs_to_copy;
7940 		fraction_done = (double)copied / total;
7941 
7942 		/* elapsed time for this pass */
7943 		elapsed = time(NULL) - prs->prs_start_time;
7944 		elapsed = elapsed > 0 ? elapsed : 1;
7945 		rate = copied / elapsed;
7946 		rate = rate > 0 ? rate : 1;
7947 		mins_left = ((total - copied) / rate) / 60;
7948 		hours_left = mins_left / 60;
7949 
7950 		zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
7951 		zfs_nicenum(total, total_buf, sizeof (total_buf));
7952 		zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
7953 
7954 		/*
7955 		 * do not print estimated time if hours_left is more than
7956 		 * 30 days
7957 		 */
7958 		(void) printf(gettext(
7959 		    "\t%s copied out of %s at %s/s, %.2f%% done"),
7960 		    examined_buf, total_buf, rate_buf, 100 * fraction_done);
7961 		if (hours_left < (30 * 24)) {
7962 			(void) printf(gettext(", %lluh%um to go\n"),
7963 			    (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
7964 		} else {
7965 			(void) printf(gettext(
7966 			    ", (copy is slow, no estimated time)\n"));
7967 		}
7968 	}
7969 	free(vdev_name);
7970 
7971 	if (prs->prs_mapping_memory > 0) {
7972 		char mem_buf[7];
7973 		zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
7974 		(void) printf(gettext(
7975 		    "\t%s memory used for removed device mappings\n"),
7976 		    mem_buf);
7977 	}
7978 }
7979 
7980 static void
7981 print_checkpoint_status(pool_checkpoint_stat_t *pcs)
7982 {
7983 	time_t start;
7984 	char space_buf[7];
7985 
7986 	if (pcs == NULL || pcs->pcs_state == CS_NONE)
7987 		return;
7988 
7989 	(void) printf(gettext("checkpoint: "));
7990 
7991 	start = pcs->pcs_start_time;
7992 	zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));
7993 
7994 	if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
7995 		char *date = ctime(&start);
7996 
7997 		/*
7998 		 * ctime() adds a newline at the end of the generated
7999 		 * string, thus the weird format specifier and the
8000 		 * strlen() call used to chop it off from the output.
8001 		 */
8002 		(void) printf(gettext("created %.*s, consumes %s\n"),
8003 		    (int)(strlen(date) - 1), date, space_buf);
8004 		return;
8005 	}
8006 
8007 	assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
8008 
8009 	(void) printf(gettext("discarding, %s remaining.\n"),
8010 	    space_buf);
8011 }
8012 
8013 static void
8014 print_error_log(zpool_handle_t *zhp)
8015 {
8016 	nvlist_t *nverrlist = NULL;
8017 	nvpair_t *elem;
8018 	char *pathname;
8019 	size_t len = MAXPATHLEN * 2;
8020 
8021 	if (zpool_get_errlog(zhp, &nverrlist) != 0)
8022 		return;
8023 
8024 	(void) printf("errors: Permanent errors have been "
8025 	    "detected in the following files:\n\n");
8026 
8027 	pathname = safe_malloc(len);
8028 	elem = NULL;
8029 	while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
8030 		nvlist_t *nv;
8031 		uint64_t dsobj, obj;
8032 
8033 		verify(nvpair_value_nvlist(elem, &nv) == 0);
8034 		verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
8035 		    &dsobj) == 0);
8036 		verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
8037 		    &obj) == 0);
8038 		zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
8039 		(void) printf("%7s %s\n", "", pathname);
8040 	}
8041 	free(pathname);
8042 	nvlist_free(nverrlist);
8043 }
8044 
8045 static void
8046 print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
8047     uint_t nspares)
8048 {
8049 	uint_t i;
8050 	char *name;
8051 
8052 	if (nspares == 0)
8053 		return;
8054 
8055 	(void) printf(gettext("\tspares\n"));
8056 
8057 	for (i = 0; i < nspares; i++) {
8058 		name = zpool_vdev_name(g_zfs, zhp, spares[i],
8059 		    cb->cb_name_flags);
8060 		print_status_config(zhp, cb, name, spares[i], 2, B_TRUE, NULL);
8061 		free(name);
8062 	}
8063 }
8064 
8065 static void
8066 print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
8067     uint_t nl2cache)
8068 {
8069 	uint_t i;
8070 	char *name;
8071 
8072 	if (nl2cache == 0)
8073 		return;
8074 
8075 	(void) printf(gettext("\tcache\n"));
8076 
8077 	for (i = 0; i < nl2cache; i++) {
8078 		name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
8079 		    cb->cb_name_flags);
8080 		print_status_config(zhp, cb, name, l2cache[i], 2,
8081 		    B_FALSE, NULL);
8082 		free(name);
8083 	}
8084 }
8085 
8086 static void
8087 print_dedup_stats(nvlist_t *config)
8088 {
8089 	ddt_histogram_t *ddh;
8090 	ddt_stat_t *dds;
8091 	ddt_object_t *ddo;
8092 	uint_t c;
8093 	char dspace[6], mspace[6];
8094 
8095 	/*
8096 	 * If the pool was faulted then we may not have been able to
8097 	 * obtain the config. Otherwise, if we have anything in the dedup
8098 	 * table continue processing the stats.
8099 	 */
8100 	if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
8101 	    (uint64_t **)&ddo, &c) != 0)
8102 		return;
8103 
8104 	(void) printf("\n");
8105 	(void) printf(gettext(" dedup: "));
8106 	if (ddo->ddo_count == 0) {
8107 		(void) printf(gettext("no DDT entries\n"));
8108 		return;
8109 	}
8110 
8111 	zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
8112 	zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
8113 	(void) printf("DDT entries %llu, size %s on disk, %s in core\n",
8114 	    (u_longlong_t)ddo->ddo_count,
8115 	    dspace,
8116 	    mspace);
8117 
8118 	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
8119 	    (uint64_t **)&dds, &c) == 0);
8120 	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
8121 	    (uint64_t **)&ddh, &c) == 0);
8122 	zpool_dump_ddt(dds, ddh);
8123 }
8124 
8125 /*
8126  * Display a summary of pool status.  Displays a summary such as:
8127  *
8128  *        pool: tank
8129  *	status: DEGRADED
8130  *	reason: One or more devices ...
8131  *         see: https://openzfs.github.io/openzfs-docs/msg/ZFS-xxxx-01
8132  *	config:
8133  *		mirror		DEGRADED
8134  *                c1t0d0	OK
8135  *                c2t0d0	UNAVAIL
8136  *
8137  * When given the '-v' option, we print out the complete config.  If the '-e'
8138  * option is specified, then we print out error rate information as well.
8139  */
8140 static int
8141 status_callback(zpool_handle_t *zhp, void *data)
8142 {
8143 	status_cbdata_t *cbp = data;
8144 	nvlist_t *config, *nvroot;
8145 	char *msgid;
8146 	zpool_status_t reason;
8147 	zpool_errata_t errata;
8148 	const char *health;
8149 	uint_t c;
8150 	vdev_stat_t *vs;
8151 
8152 	config = zpool_get_config(zhp, NULL);
8153 	reason = zpool_get_status(zhp, &msgid, &errata);
8154 
8155 	cbp->cb_count++;
8156 
8157 	/*
8158 	 * If we were given 'zpool status -x', only report those pools with
8159 	 * problems.
8160 	 */
8161 	if (cbp->cb_explain &&
8162 	    (reason == ZPOOL_STATUS_OK ||
8163 	    reason == ZPOOL_STATUS_VERSION_OLDER ||
8164 	    reason == ZPOOL_STATUS_FEAT_DISABLED ||
8165 	    reason == ZPOOL_STATUS_COMPATIBILITY_ERR ||
8166 	    reason == ZPOOL_STATUS_INCOMPATIBLE_FEAT)) {
8167 		if (!cbp->cb_allpools) {
8168 			(void) printf(gettext("pool '%s' is healthy\n"),
8169 			    zpool_get_name(zhp));
8170 			if (cbp->cb_first)
8171 				cbp->cb_first = B_FALSE;
8172 		}
8173 		return (0);
8174 	}
8175 
8176 	if (cbp->cb_first)
8177 		cbp->cb_first = B_FALSE;
8178 	else
8179 		(void) printf("\n");
8180 
8181 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
8182 	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
8183 	    (uint64_t **)&vs, &c) == 0);
8184 
8185 	health = zpool_get_state_str(zhp);
8186 
8187 	printf("  ");
8188 	printf_color(ANSI_BOLD, gettext("pool:"));
8189 	printf(" %s\n", zpool_get_name(zhp));
8190 	printf(" ");
8191 	printf_color(ANSI_BOLD, gettext("state: "));
8192 
8193 	printf_color(health_str_to_color(health), "%s", health);
8194 
8195 	printf("\n");
8196 
8197 	switch (reason) {
8198 	case ZPOOL_STATUS_MISSING_DEV_R:
8199 		printf_color(ANSI_BOLD, gettext("status: "));
8200 		printf_color(ANSI_YELLOW, gettext("One or more devices could "
8201 		    "not be opened.  Sufficient replicas exist for\n\tthe pool "
8202 		    "to continue functioning in a degraded state.\n"));
8203 		printf_color(ANSI_BOLD, gettext("action: "));
8204 		printf_color(ANSI_YELLOW, gettext("Attach the missing device "
8205 		    "and online it using 'zpool online'.\n"));
8206 		break;
8207 
8208 	case ZPOOL_STATUS_MISSING_DEV_NR:
8209 		printf_color(ANSI_BOLD, gettext("status: "));
8210 		printf_color(ANSI_YELLOW, gettext("One or more devices could "
8211 		    "not be opened.  There are insufficient\n\treplicas for the"
8212 		    " pool to continue functioning.\n"));
8213 		printf_color(ANSI_BOLD, gettext("action: "));
8214 		printf_color(ANSI_YELLOW, gettext("Attach the missing device "
8215 		    "and online it using 'zpool online'.\n"));
8216 		break;
8217 
8218 	case ZPOOL_STATUS_CORRUPT_LABEL_R:
8219 		printf_color(ANSI_BOLD, gettext("status: "));
8220 		printf_color(ANSI_YELLOW, gettext("One or more devices could "
8221 		    "not be used because the label is missing or\n\tinvalid.  "
8222 		    "Sufficient replicas exist for the pool to continue\n\t"
8223 		    "functioning in a degraded state.\n"));
8224 		printf_color(ANSI_BOLD, gettext("action: "));
8225 		printf_color(ANSI_YELLOW, gettext("Replace the device using "
8226 		    "'zpool replace'.\n"));
8227 		break;
8228 
8229 	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
8230 		printf_color(ANSI_BOLD, gettext("status: "));
8231 		printf_color(ANSI_YELLOW, gettext("One or more devices could "
8232 		    "not be used because the label is missing \n\tor invalid.  "
8233 		    "There are insufficient replicas for the pool to "
8234 		    "continue\n\tfunctioning.\n"));
8235 		zpool_explain_recover(zpool_get_handle(zhp),
8236 		    zpool_get_name(zhp), reason, config);
8237 		break;
8238 
8239 	case ZPOOL_STATUS_FAILING_DEV:
8240 		printf_color(ANSI_BOLD, gettext("status: "));
8241 		printf_color(ANSI_YELLOW, gettext("One or more devices has "
8242 		    "experienced an unrecoverable error.  An\n\tattempt was "
8243 		    "made to correct the error.  Applications are "
8244 		    "unaffected.\n"));
8245 		printf_color(ANSI_BOLD, gettext("action: "));
8246 			printf_color(ANSI_YELLOW, gettext("Determine if the "
8247 		    "device needs to be replaced, and clear the errors\n\tusing"
8248 		    " 'zpool clear' or replace the device with 'zpool "
8249 		    "replace'.\n"));
8250 		break;
8251 
8252 	case ZPOOL_STATUS_OFFLINE_DEV:
8253 		printf_color(ANSI_BOLD, gettext("status: "));
8254 		printf_color(ANSI_YELLOW, gettext("One or more devices has "
8255 		    "been taken offline by the administrator.\n\tSufficient "
8256 		    "replicas exist for the pool to continue functioning in "
8257 		    "a\n\tdegraded state.\n"));
8258 		printf_color(ANSI_BOLD, gettext("action: "));
8259 		printf_color(ANSI_YELLOW, gettext("Online the device "
8260 		    "using 'zpool online' or replace the device with\n\t'zpool "
8261 		    "replace'.\n"));
8262 		break;
8263 
8264 	case ZPOOL_STATUS_REMOVED_DEV:
8265 		printf_color(ANSI_BOLD, gettext("status: "));
8266 		printf_color(ANSI_YELLOW, gettext("One or more devices has "
8267 		    "been removed by the administrator.\n\tSufficient "
8268 		    "replicas exist for the pool to continue functioning in "
8269 		    "a\n\tdegraded state.\n"));
8270 		printf_color(ANSI_BOLD, gettext("action: "));
8271 		printf_color(ANSI_YELLOW, gettext("Online the device "
8272 		    "using zpool online' or replace the device with\n\t'zpool "
8273 		    "replace'.\n"));
8274 		break;
8275 
8276 	case ZPOOL_STATUS_RESILVERING:
8277 	case ZPOOL_STATUS_REBUILDING:
8278 		printf_color(ANSI_BOLD, gettext("status: "));
8279 		printf_color(ANSI_YELLOW, gettext("One or more devices is "
8280 		    "currently being resilvered.  The pool will\n\tcontinue "
8281 		    "to function, possibly in a degraded state.\n"));
8282 		printf_color(ANSI_BOLD, gettext("action: "));
8283 		printf_color(ANSI_YELLOW, gettext("Wait for the resilver to "
8284 		    "complete.\n"));
8285 		break;
8286 
8287 	case ZPOOL_STATUS_REBUILD_SCRUB:
8288 		printf_color(ANSI_BOLD, gettext("status: "));
8289 		printf_color(ANSI_YELLOW, gettext("One or more devices have "
8290 		    "been sequentially resilvered, scrubbing\n\tthe pool "
8291 		    "is recommended.\n"));
8292 		printf_color(ANSI_BOLD, gettext("action: "));
8293 		printf_color(ANSI_YELLOW, gettext("Use 'zpool scrub' to "
8294 		    "verify all data checksums.\n"));
8295 		break;
8296 
8297 	case ZPOOL_STATUS_CORRUPT_DATA:
8298 		printf_color(ANSI_BOLD, gettext("status: "));
8299 		printf_color(ANSI_YELLOW, gettext("One or more devices has "
8300 		    "experienced an error resulting in data\n\tcorruption.  "
8301 		    "Applications may be affected.\n"));
8302 		printf_color(ANSI_BOLD, gettext("action: "));
8303 		printf_color(ANSI_YELLOW, gettext("Restore the file in question"
8304 		    " if possible.  Otherwise restore the\n\tentire pool from "
8305 		    "backup.\n"));
8306 		break;
8307 
8308 	case ZPOOL_STATUS_CORRUPT_POOL:
8309 		printf_color(ANSI_BOLD, gettext("status: "));
8310 		printf_color(ANSI_YELLOW, gettext("The pool metadata is "
8311 		    "corrupted and the pool cannot be opened.\n"));
8312 		zpool_explain_recover(zpool_get_handle(zhp),
8313 		    zpool_get_name(zhp), reason, config);
8314 		break;
8315 
8316 	case ZPOOL_STATUS_VERSION_OLDER:
8317 		printf_color(ANSI_BOLD, gettext("status: "));
8318 		printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
8319 		    "a legacy on-disk format.  The pool can\n\tstill be used, "
8320 		    "but some features are unavailable.\n"));
8321 		printf_color(ANSI_BOLD, gettext("action: "));
8322 		printf_color(ANSI_YELLOW, gettext("Upgrade the pool using "
8323 		    "'zpool upgrade'.  Once this is done, the\n\tpool will no "
8324 		    "longer be accessible on software that does not support\n\t"
8325 		    "feature flags.\n"));
8326 		break;
8327 
8328 	case ZPOOL_STATUS_VERSION_NEWER:
8329 		printf_color(ANSI_BOLD, gettext("status: "));
8330 		printf_color(ANSI_YELLOW, gettext("The pool has been upgraded "
8331 		    "to a newer, incompatible on-disk version.\n\tThe pool "
8332 		    "cannot be accessed on this system.\n"));
8333 		printf_color(ANSI_BOLD, gettext("action: "));
8334 		printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8335 		    "system running more recent software, or\n\trestore the "
8336 		    "pool from backup.\n"));
8337 		break;
8338 
8339 	case ZPOOL_STATUS_FEAT_DISABLED:
8340 		printf_color(ANSI_BOLD, gettext("status: "));
8341 		printf_color(ANSI_YELLOW, gettext("Some supported and "
8342 		    "requested features are not enabled on the pool.\n\t"
8343 		    "The pool can still be used, but some features are "
8344 		    "unavailable.\n"));
8345 		printf_color(ANSI_BOLD, gettext("action: "));
8346 		printf_color(ANSI_YELLOW, gettext("Enable all features using "
8347 		    "'zpool upgrade'. Once this is done,\n\tthe pool may no "
8348 		    "longer be accessible by software that does not support\n\t"
8349 		    "the features. See zpool-features(7) for details.\n"));
8350 		break;
8351 
8352 	case ZPOOL_STATUS_COMPATIBILITY_ERR:
8353 		printf_color(ANSI_BOLD, gettext("status: "));
8354 		printf_color(ANSI_YELLOW, gettext("This pool has a "
8355 		    "compatibility list specified, but it could not be\n\t"
8356 		    "read/parsed at this time. The pool can still be used, "
8357 		    "but this\n\tshould be investigated.\n"));
8358 		printf_color(ANSI_BOLD, gettext("action: "));
8359 		printf_color(ANSI_YELLOW, gettext("Check the value of the "
8360 		    "'compatibility' property against the\n\t"
8361 		    "appropriate file in " ZPOOL_SYSCONF_COMPAT_D " or "
8362 		    ZPOOL_DATA_COMPAT_D ".\n"));
8363 		break;
8364 
8365 	case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
8366 		printf_color(ANSI_BOLD, gettext("status: "));
8367 		printf_color(ANSI_YELLOW, gettext("One or more features "
8368 		    "are enabled on the pool despite not being\n\t"
8369 		    "requested by the 'compatibility' property.\n"));
8370 		printf_color(ANSI_BOLD, gettext("action: "));
8371 		printf_color(ANSI_YELLOW, gettext("Consider setting "
8372 		    "'compatibility' to an appropriate value, or\n\t"
8373 		    "adding needed features to the relevant file in\n\t"
8374 		    ZPOOL_SYSCONF_COMPAT_D " or " ZPOOL_DATA_COMPAT_D ".\n"));
8375 		break;
8376 
8377 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
8378 		printf_color(ANSI_BOLD, gettext("status: "));
8379 		printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8380 		    "on this system because it uses the\n\tfollowing feature(s)"
8381 		    " not supported on this system:\n"));
8382 		zpool_print_unsup_feat(config);
8383 		(void) printf("\n");
8384 		printf_color(ANSI_BOLD, gettext("action: "));
8385 		printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8386 		    "system that supports the required feature(s),\n\tor "
8387 		    "restore the pool from backup.\n"));
8388 		break;
8389 
8390 	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
8391 		printf_color(ANSI_BOLD, gettext("status: "));
8392 		printf_color(ANSI_YELLOW, gettext("The pool can only be "
8393 		    "accessed in read-only mode on this system. It\n\tcannot be"
8394 		    " accessed in read-write mode because it uses the "
8395 		    "following\n\tfeature(s) not supported on this system:\n"));
8396 		zpool_print_unsup_feat(config);
8397 		(void) printf("\n");
8398 		printf_color(ANSI_BOLD, gettext("action: "));
8399 		printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8400 		    "in read-write mode. Import the pool with\n"
8401 		    "\t\"-o readonly=on\", access the pool from a system that "
8402 		    "supports the\n\trequired feature(s), or restore the "
8403 		    "pool from backup.\n"));
8404 		break;
8405 
8406 	case ZPOOL_STATUS_FAULTED_DEV_R:
8407 		printf_color(ANSI_BOLD, gettext("status: "));
8408 		printf_color(ANSI_YELLOW, gettext("One or more devices are "
8409 		    "faulted in response to persistent errors.\n\tSufficient "
8410 		    "replicas exist for the pool to continue functioning "
8411 		    "in a\n\tdegraded state.\n"));
8412 		printf_color(ANSI_BOLD, gettext("action: "));
8413 		printf_color(ANSI_YELLOW, gettext("Replace the faulted device, "
8414 		    "or use 'zpool clear' to mark the device\n\trepaired.\n"));
8415 		break;
8416 
8417 	case ZPOOL_STATUS_FAULTED_DEV_NR:
8418 		printf_color(ANSI_BOLD, gettext("status: "));
8419 		printf_color(ANSI_YELLOW, gettext("One or more devices are "
8420 		    "faulted in response to persistent errors.  There are "
8421 		    "insufficient replicas for the pool to\n\tcontinue "
8422 		    "functioning.\n"));
8423 		printf_color(ANSI_BOLD, gettext("action: "));
8424 		printf_color(ANSI_YELLOW, gettext("Destroy and re-create the "
8425 		    "pool from a backup source.  Manually marking the device\n"
8426 		    "\trepaired using 'zpool clear' may allow some data "
8427 		    "to be recovered.\n"));
8428 		break;
8429 
8430 	case ZPOOL_STATUS_IO_FAILURE_MMP:
8431 		printf_color(ANSI_BOLD, gettext("status: "));
8432 		printf_color(ANSI_YELLOW, gettext("The pool is suspended "
8433 		    "because multihost writes failed or were delayed;\n\t"
8434 		    "another system could import the pool undetected.\n"));
8435 		printf_color(ANSI_BOLD, gettext("action: "));
8436 		printf_color(ANSI_YELLOW, gettext("Make sure the pool's devices"
8437 		    " are connected, then reboot your system and\n\timport the "
8438 		    "pool.\n"));
8439 		break;
8440 
8441 	case ZPOOL_STATUS_IO_FAILURE_WAIT:
8442 	case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
8443 		printf_color(ANSI_BOLD, gettext("status: "));
8444 		printf_color(ANSI_YELLOW, gettext("One or more devices are "
8445 		    "faulted in response to IO failures.\n"));
8446 		printf_color(ANSI_BOLD, gettext("action: "));
8447 		printf_color(ANSI_YELLOW, gettext("Make sure the affected "
8448 		    "devices are connected, then run 'zpool clear'.\n"));
8449 		break;
8450 
8451 	case ZPOOL_STATUS_BAD_LOG:
8452 		printf_color(ANSI_BOLD, gettext("status: "));
8453 		printf_color(ANSI_YELLOW, gettext("An intent log record "
8454 		    "could not be read.\n"
8455 		    "\tWaiting for administrator intervention to fix the "
8456 		    "faulted pool.\n"));
8457 		printf_color(ANSI_BOLD, gettext("action: "));
8458 		printf_color(ANSI_YELLOW, gettext("Either restore the affected "
8459 		    "device(s) and run 'zpool online',\n"
8460 		    "\tor ignore the intent log records by running "
8461 		    "'zpool clear'.\n"));
8462 		break;
8463 
8464 	case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
8465 		(void) printf(gettext("status: One or more devices are "
8466 		    "configured to use a non-native block size.\n"
8467 		    "\tExpect reduced performance.\n"));
8468 		(void) printf(gettext("action: Replace affected devices with "
8469 		    "devices that support the\n\tconfigured block size, or "
8470 		    "migrate data to a properly configured\n\tpool.\n"));
8471 		break;
8472 
8473 	case ZPOOL_STATUS_HOSTID_MISMATCH:
8474 		printf_color(ANSI_BOLD, gettext("status: "));
8475 		printf_color(ANSI_YELLOW, gettext("Mismatch between pool hostid"
8476 		    " and system hostid on imported pool.\n\tThis pool was "
8477 		    "previously imported into a system with a different "
8478 		    "hostid,\n\tand then was verbatim imported into this "
8479 		    "system.\n"));
8480 		printf_color(ANSI_BOLD, gettext("action: "));
8481 		printf_color(ANSI_YELLOW, gettext("Export this pool on all "
8482 		    "systems on which it is imported.\n"
8483 		    "\tThen import it to correct the mismatch.\n"));
8484 		break;
8485 
8486 	case ZPOOL_STATUS_ERRATA:
8487 		printf_color(ANSI_BOLD, gettext("status: "));
8488 		printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
8489 		    errata);
8490 
8491 		switch (errata) {
8492 		case ZPOOL_ERRATA_NONE:
8493 			break;
8494 
8495 		case ZPOOL_ERRATA_ZOL_2094_SCRUB:
8496 			printf_color(ANSI_BOLD, gettext("action: "));
8497 			printf_color(ANSI_YELLOW, gettext("To correct the issue"
8498 			    " run 'zpool scrub'.\n"));
8499 			break;
8500 
8501 		case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
8502 			(void) printf(gettext("\tExisting encrypted datasets "
8503 			    "contain an on-disk incompatibility\n\twhich "
8504 			    "needs to be corrected.\n"));
8505 			printf_color(ANSI_BOLD, gettext("action: "));
8506 			printf_color(ANSI_YELLOW, gettext("To correct the issue"
8507 			    " backup existing encrypted datasets to new\n\t"
8508 			    "encrypted datasets and destroy the old ones. "
8509 			    "'zfs mount -o ro' can\n\tbe used to temporarily "
8510 			    "mount existing encrypted datasets readonly.\n"));
8511 			break;
8512 
8513 		case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
8514 			(void) printf(gettext("\tExisting encrypted snapshots "
8515 			    "and bookmarks contain an on-disk\n\tincompat"
8516 			    "ibility. This may cause on-disk corruption if "
8517 			    "they are used\n\twith 'zfs recv'.\n"));
8518 			printf_color(ANSI_BOLD, gettext("action: "));
8519 			printf_color(ANSI_YELLOW, gettext("To correct the"
8520 			    "issue, enable the bookmark_v2 feature. No "
8521 			    "additional\n\taction is needed if there are no "
8522 			    "encrypted snapshots or bookmarks.\n\tIf preserving"
8523 			    "the encrypted snapshots and bookmarks is required,"
8524 			    " use\n\ta non-raw send to backup and restore them."
8525 			    " Alternately, they may be\n\tremoved to resolve "
8526 			    "the incompatibility.\n"));
8527 			break;
8528 
8529 		default:
8530 			/*
8531 			 * All errata which allow the pool to be imported
8532 			 * must contain an action message.
8533 			 */
8534 			assert(0);
8535 		}
8536 		break;
8537 
8538 	default:
8539 		/*
8540 		 * The remaining errors can't actually be generated, yet.
8541 		 */
8542 		assert(reason == ZPOOL_STATUS_OK);
8543 	}
8544 
8545 	if (msgid != NULL) {
8546 		printf("   ");
8547 		printf_color(ANSI_BOLD, gettext("see:"));
8548 		printf(gettext(
8549 		    " https://openzfs.github.io/openzfs-docs/msg/%s\n"),
8550 		    msgid);
8551 	}
8552 
8553 	if (config != NULL) {
8554 		uint64_t nerr;
8555 		nvlist_t **spares, **l2cache;
8556 		uint_t nspares, nl2cache;
8557 		pool_checkpoint_stat_t *pcs = NULL;
8558 		pool_removal_stat_t *prs = NULL;
8559 
8560 		print_scan_status(zhp, nvroot);
8561 
8562 		(void) nvlist_lookup_uint64_array(nvroot,
8563 		    ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
8564 		print_removal_status(zhp, prs);
8565 
8566 		(void) nvlist_lookup_uint64_array(nvroot,
8567 		    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
8568 		print_checkpoint_status(pcs);
8569 
8570 		cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
8571 		    cbp->cb_name_flags | VDEV_NAME_TYPE_ID);
8572 		if (cbp->cb_namewidth < 10)
8573 			cbp->cb_namewidth = 10;
8574 
8575 		color_start(ANSI_BOLD);
8576 		(void) printf(gettext("config:\n\n"));
8577 		(void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
8578 		    cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
8579 		    "CKSUM");
8580 		color_end();
8581 
8582 		if (cbp->cb_print_slow_ios) {
8583 			printf_color(ANSI_BOLD, " %5s", gettext("SLOW"));
8584 		}
8585 
8586 		if (cbp->vcdl != NULL)
8587 			print_cmd_columns(cbp->vcdl, 0);
8588 
8589 		printf("\n");
8590 
8591 		print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
8592 		    B_FALSE, NULL);
8593 
8594 		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_DEDUP);
8595 		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
8596 		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_CLASS_LOGS);
8597 
8598 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
8599 		    &l2cache, &nl2cache) == 0)
8600 			print_l2cache(zhp, cbp, l2cache, nl2cache);
8601 
8602 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
8603 		    &spares, &nspares) == 0)
8604 			print_spares(zhp, cbp, spares, nspares);
8605 
8606 		if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
8607 		    &nerr) == 0) {
8608 			nvlist_t *nverrlist = NULL;
8609 
8610 			/*
8611 			 * If the approximate error count is small, get a
8612 			 * precise count by fetching the entire log and
8613 			 * uniquifying the results.
8614 			 */
8615 			if (nerr > 0 && nerr < 100 && !cbp->cb_verbose &&
8616 			    zpool_get_errlog(zhp, &nverrlist) == 0) {
8617 				nvpair_t *elem;
8618 
8619 				elem = NULL;
8620 				nerr = 0;
8621 				while ((elem = nvlist_next_nvpair(nverrlist,
8622 				    elem)) != NULL) {
8623 					nerr++;
8624 				}
8625 			}
8626 			nvlist_free(nverrlist);
8627 
8628 			(void) printf("\n");
8629 
8630 			if (nerr == 0)
8631 				(void) printf(gettext("errors: No known data "
8632 				    "errors\n"));
8633 			else if (!cbp->cb_verbose)
8634 				(void) printf(gettext("errors: %llu data "
8635 				    "errors, use '-v' for a list\n"),
8636 				    (u_longlong_t)nerr);
8637 			else
8638 				print_error_log(zhp);
8639 		}
8640 
8641 		if (cbp->cb_dedup_stats)
8642 			print_dedup_stats(config);
8643 	} else {
8644 		(void) printf(gettext("config: The configuration cannot be "
8645 		    "determined.\n"));
8646 	}
8647 
8648 	return (0);
8649 }
8650 
8651 /*
8652  * zpool status [-c [script1,script2,...]] [-igLpPstvx] [-T d|u] [pool] ...
8653  *              [interval [count]]
8654  *
8655  *	-c CMD	For each vdev, run command CMD
8656  *	-i	Display vdev initialization status.
8657  *	-g	Display guid for individual vdev name.
8658  *	-L	Follow links when resolving vdev path name.
8659  *	-p	Display values in parsable (exact) format.
8660  *	-P	Display full path for vdev name.
8661  *	-s	Display slow IOs column.
8662  *	-v	Display complete error logs
8663  *	-x	Display only pools with potential problems
8664  *	-D	Display dedup status (undocumented)
8665  *	-t	Display vdev TRIM status.
8666  *	-T	Display a timestamp in date(1) or Unix format
8667  *
8668  * Describes the health status of all pools or some subset.
8669  */
8670 int
8671 zpool_do_status(int argc, char **argv)
8672 {
8673 	int c;
8674 	int ret;
8675 	float interval = 0;
8676 	unsigned long count = 0;
8677 	status_cbdata_t cb = { 0 };
8678 	char *cmd = NULL;
8679 
8680 	/* check options */
8681 	while ((c = getopt(argc, argv, "c:igLpPsvxDtT:")) != -1) {
8682 		switch (c) {
8683 		case 'c':
8684 			if (cmd != NULL) {
8685 				fprintf(stderr,
8686 				    gettext("Can't set -c flag twice\n"));
8687 				exit(1);
8688 			}
8689 
8690 			if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
8691 			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
8692 				fprintf(stderr, gettext(
8693 				    "Can't run -c, disabled by "
8694 				    "ZPOOL_SCRIPTS_ENABLED.\n"));
8695 				exit(1);
8696 			}
8697 
8698 			if ((getuid() <= 0 || geteuid() <= 0) &&
8699 			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
8700 				fprintf(stderr, gettext(
8701 				    "Can't run -c with root privileges "
8702 				    "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
8703 				exit(1);
8704 			}
8705 			cmd = optarg;
8706 			break;
8707 		case 'i':
8708 			cb.cb_print_vdev_init = B_TRUE;
8709 			break;
8710 		case 'g':
8711 			cb.cb_name_flags |= VDEV_NAME_GUID;
8712 			break;
8713 		case 'L':
8714 			cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
8715 			break;
8716 		case 'p':
8717 			cb.cb_literal = B_TRUE;
8718 			break;
8719 		case 'P':
8720 			cb.cb_name_flags |= VDEV_NAME_PATH;
8721 			break;
8722 		case 's':
8723 			cb.cb_print_slow_ios = B_TRUE;
8724 			break;
8725 		case 'v':
8726 			cb.cb_verbose = B_TRUE;
8727 			break;
8728 		case 'x':
8729 			cb.cb_explain = B_TRUE;
8730 			break;
8731 		case 'D':
8732 			cb.cb_dedup_stats = B_TRUE;
8733 			break;
8734 		case 't':
8735 			cb.cb_print_vdev_trim = B_TRUE;
8736 			break;
8737 		case 'T':
8738 			get_timestamp_arg(*optarg);
8739 			break;
8740 		case '?':
8741 			if (optopt == 'c') {
8742 				print_zpool_script_list("status");
8743 				exit(0);
8744 			} else {
8745 				fprintf(stderr,
8746 				    gettext("invalid option '%c'\n"), optopt);
8747 			}
8748 			usage(B_FALSE);
8749 		}
8750 	}
8751 
8752 	argc -= optind;
8753 	argv += optind;
8754 
8755 	get_interval_count(&argc, argv, &interval, &count);
8756 
8757 	if (argc == 0)
8758 		cb.cb_allpools = B_TRUE;
8759 
8760 	cb.cb_first = B_TRUE;
8761 	cb.cb_print_status = B_TRUE;
8762 
8763 	for (;;) {
8764 		if (timestamp_fmt != NODATE)
8765 			print_timestamp(timestamp_fmt);
8766 
8767 		if (cmd != NULL)
8768 			cb.vcdl = all_pools_for_each_vdev_run(argc, argv, cmd,
8769 			    NULL, NULL, 0, 0);
8770 
8771 		ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
8772 		    cb.cb_literal, status_callback, &cb);
8773 
8774 		if (cb.vcdl != NULL)
8775 			free_vdev_cmd_data_list(cb.vcdl);
8776 
8777 		if (argc == 0 && cb.cb_count == 0)
8778 			(void) fprintf(stderr, gettext("no pools available\n"));
8779 		else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
8780 			(void) printf(gettext("all pools are healthy\n"));
8781 
8782 		if (ret != 0)
8783 			return (ret);
8784 
8785 		if (interval == 0)
8786 			break;
8787 
8788 		if (count != 0 && --count == 0)
8789 			break;
8790 
8791 		(void) fsleep(interval);
8792 	}
8793 
8794 	return (0);
8795 }
8796 
8797 typedef struct upgrade_cbdata {
8798 	int	cb_first;
8799 	int	cb_argc;
8800 	uint64_t cb_version;
8801 	char	**cb_argv;
8802 } upgrade_cbdata_t;
8803 
8804 static int
8805 check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs)
8806 {
8807 	int zfs_version = (int)zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
8808 	int *count = (int *)unsupp_fs;
8809 
8810 	if (zfs_version > ZPL_VERSION) {
8811 		(void) printf(gettext("%s (v%d) is not supported by this "
8812 		    "implementation of ZFS.\n"),
8813 		    zfs_get_name(zhp), zfs_version);
8814 		(*count)++;
8815 	}
8816 
8817 	zfs_iter_filesystems(zhp, check_unsupp_fs, unsupp_fs);
8818 
8819 	zfs_close(zhp);
8820 
8821 	return (0);
8822 }
8823 
8824 static int
8825 upgrade_version(zpool_handle_t *zhp, uint64_t version)
8826 {
8827 	int ret;
8828 	nvlist_t *config;
8829 	uint64_t oldversion;
8830 	int unsupp_fs = 0;
8831 
8832 	config = zpool_get_config(zhp, NULL);
8833 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8834 	    &oldversion) == 0);
8835 
8836 	char compat[ZFS_MAXPROPLEN];
8837 	if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
8838 	    ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
8839 		compat[0] = '\0';
8840 
8841 	assert(SPA_VERSION_IS_SUPPORTED(oldversion));
8842 	assert(oldversion < version);
8843 
8844 	ret = zfs_iter_root(zpool_get_handle(zhp), check_unsupp_fs, &unsupp_fs);
8845 	if (ret != 0)
8846 		return (ret);
8847 
8848 	if (unsupp_fs) {
8849 		(void) fprintf(stderr, gettext("Upgrade not performed due "
8850 		    "to %d unsupported filesystems (max v%d).\n"),
8851 		    unsupp_fs, (int)ZPL_VERSION);
8852 		return (1);
8853 	}
8854 
8855 	if (strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0) {
8856 		(void) fprintf(stderr, gettext("Upgrade not performed because "
8857 		    "'compatibility' property set to '"
8858 		    ZPOOL_COMPAT_LEGACY "'.\n"));
8859 		return (1);
8860 	}
8861 
8862 	ret = zpool_upgrade(zhp, version);
8863 	if (ret != 0)
8864 		return (ret);
8865 
8866 	if (version >= SPA_VERSION_FEATURES) {
8867 		(void) printf(gettext("Successfully upgraded "
8868 		    "'%s' from version %llu to feature flags.\n"),
8869 		    zpool_get_name(zhp), (u_longlong_t)oldversion);
8870 	} else {
8871 		(void) printf(gettext("Successfully upgraded "
8872 		    "'%s' from version %llu to version %llu.\n"),
8873 		    zpool_get_name(zhp), (u_longlong_t)oldversion,
8874 		    (u_longlong_t)version);
8875 	}
8876 
8877 	return (0);
8878 }
8879 
8880 static int
8881 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
8882 {
8883 	int i, ret, count;
8884 	boolean_t firstff = B_TRUE;
8885 	nvlist_t *enabled = zpool_get_features(zhp);
8886 
8887 	char compat[ZFS_MAXPROPLEN];
8888 	if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
8889 	    ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
8890 		compat[0] = '\0';
8891 
8892 	boolean_t requested_features[SPA_FEATURES];
8893 	if (zpool_do_load_compat(compat, requested_features) !=
8894 	    ZPOOL_COMPATIBILITY_OK)
8895 		return (-1);
8896 
8897 	count = 0;
8898 	for (i = 0; i < SPA_FEATURES; i++) {
8899 		const char *fname = spa_feature_table[i].fi_uname;
8900 		const char *fguid = spa_feature_table[i].fi_guid;
8901 
8902 		if (!spa_feature_table[i].fi_zfs_mod_supported)
8903 			continue;
8904 
8905 		if (!nvlist_exists(enabled, fguid) && requested_features[i]) {
8906 			char *propname;
8907 			verify(-1 != asprintf(&propname, "feature@%s", fname));
8908 			ret = zpool_set_prop(zhp, propname,
8909 			    ZFS_FEATURE_ENABLED);
8910 			if (ret != 0) {
8911 				free(propname);
8912 				return (ret);
8913 			}
8914 			count++;
8915 
8916 			if (firstff) {
8917 				(void) printf(gettext("Enabled the "
8918 				    "following features on '%s':\n"),
8919 				    zpool_get_name(zhp));
8920 				firstff = B_FALSE;
8921 			}
8922 			(void) printf(gettext("  %s\n"), fname);
8923 			free(propname);
8924 		}
8925 	}
8926 
8927 	if (countp != NULL)
8928 		*countp = count;
8929 	return (0);
8930 }
8931 
8932 static int
8933 upgrade_cb(zpool_handle_t *zhp, void *arg)
8934 {
8935 	upgrade_cbdata_t *cbp = arg;
8936 	nvlist_t *config;
8937 	uint64_t version;
8938 	boolean_t modified_pool = B_FALSE;
8939 	int ret;
8940 
8941 	config = zpool_get_config(zhp, NULL);
8942 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8943 	    &version) == 0);
8944 
8945 	assert(SPA_VERSION_IS_SUPPORTED(version));
8946 
8947 	if (version < cbp->cb_version) {
8948 		cbp->cb_first = B_FALSE;
8949 		ret = upgrade_version(zhp, cbp->cb_version);
8950 		if (ret != 0)
8951 			return (ret);
8952 		modified_pool = B_TRUE;
8953 
8954 		/*
8955 		 * If they did "zpool upgrade -a", then we could
8956 		 * be doing ioctls to different pools.  We need
8957 		 * to log this history once to each pool, and bypass
8958 		 * the normal history logging that happens in main().
8959 		 */
8960 		(void) zpool_log_history(g_zfs, history_str);
8961 		log_history = B_FALSE;
8962 	}
8963 
8964 	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
8965 		int count;
8966 		ret = upgrade_enable_all(zhp, &count);
8967 		if (ret != 0)
8968 			return (ret);
8969 
8970 		if (count > 0) {
8971 			cbp->cb_first = B_FALSE;
8972 			modified_pool = B_TRUE;
8973 		}
8974 	}
8975 
8976 	if (modified_pool) {
8977 		(void) printf("\n");
8978 		(void) after_zpool_upgrade(zhp);
8979 	}
8980 
8981 	return (0);
8982 }
8983 
8984 static int
8985 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
8986 {
8987 	upgrade_cbdata_t *cbp = arg;
8988 	nvlist_t *config;
8989 	uint64_t version;
8990 
8991 	config = zpool_get_config(zhp, NULL);
8992 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8993 	    &version) == 0);
8994 
8995 	assert(SPA_VERSION_IS_SUPPORTED(version));
8996 
8997 	if (version < SPA_VERSION_FEATURES) {
8998 		if (cbp->cb_first) {
8999 			(void) printf(gettext("The following pools are "
9000 			    "formatted with legacy version numbers and can\n"
9001 			    "be upgraded to use feature flags.  After "
9002 			    "being upgraded, these pools\nwill no "
9003 			    "longer be accessible by software that does not "
9004 			    "support feature\nflags.\n\n"
9005 			    "Note that setting a pool's 'compatibility' "
9006 			    "feature to '" ZPOOL_COMPAT_LEGACY "' will\n"
9007 			    "inhibit upgrades.\n\n"));
9008 			(void) printf(gettext("VER  POOL\n"));
9009 			(void) printf(gettext("---  ------------\n"));
9010 			cbp->cb_first = B_FALSE;
9011 		}
9012 
9013 		(void) printf("%2llu   %s\n", (u_longlong_t)version,
9014 		    zpool_get_name(zhp));
9015 	}
9016 
9017 	return (0);
9018 }
9019 
9020 static int
9021 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
9022 {
9023 	upgrade_cbdata_t *cbp = arg;
9024 	nvlist_t *config;
9025 	uint64_t version;
9026 
9027 	config = zpool_get_config(zhp, NULL);
9028 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9029 	    &version) == 0);
9030 
9031 	if (version >= SPA_VERSION_FEATURES) {
9032 		int i;
9033 		boolean_t poolfirst = B_TRUE;
9034 		nvlist_t *enabled = zpool_get_features(zhp);
9035 
9036 		for (i = 0; i < SPA_FEATURES; i++) {
9037 			const char *fguid = spa_feature_table[i].fi_guid;
9038 			const char *fname = spa_feature_table[i].fi_uname;
9039 
9040 			if (!spa_feature_table[i].fi_zfs_mod_supported)
9041 				continue;
9042 
9043 			if (!nvlist_exists(enabled, fguid)) {
9044 				if (cbp->cb_first) {
9045 					(void) printf(gettext("\nSome "
9046 					    "supported features are not "
9047 					    "enabled on the following pools. "
9048 					    "Once a\nfeature is enabled the "
9049 					    "pool may become incompatible with "
9050 					    "software\nthat does not support "
9051 					    "the feature. See "
9052 					    "zpool-features(7) for "
9053 					    "details.\n\n"
9054 					    "Note that the pool "
9055 					    "'compatibility' feature can be "
9056 					    "used to inhibit\nfeature "
9057 					    "upgrades.\n\n"));
9058 					(void) printf(gettext("POOL  "
9059 					    "FEATURE\n"));
9060 					(void) printf(gettext("------"
9061 					    "---------\n"));
9062 					cbp->cb_first = B_FALSE;
9063 				}
9064 
9065 				if (poolfirst) {
9066 					(void) printf(gettext("%s\n"),
9067 					    zpool_get_name(zhp));
9068 					poolfirst = B_FALSE;
9069 				}
9070 
9071 				(void) printf(gettext("      %s\n"), fname);
9072 			}
9073 			/*
9074 			 * If they did "zpool upgrade -a", then we could
9075 			 * be doing ioctls to different pools.  We need
9076 			 * to log this history once to each pool, and bypass
9077 			 * the normal history logging that happens in main().
9078 			 */
9079 			(void) zpool_log_history(g_zfs, history_str);
9080 			log_history = B_FALSE;
9081 		}
9082 	}
9083 
9084 	return (0);
9085 }
9086 
9087 /* ARGSUSED */
9088 static int
9089 upgrade_one(zpool_handle_t *zhp, void *data)
9090 {
9091 	boolean_t modified_pool = B_FALSE;
9092 	upgrade_cbdata_t *cbp = data;
9093 	uint64_t cur_version;
9094 	int ret;
9095 
9096 	if (strcmp("log", zpool_get_name(zhp)) == 0) {
9097 		(void) fprintf(stderr, gettext("'log' is now a reserved word\n"
9098 		    "Pool 'log' must be renamed using export and import"
9099 		    " to upgrade.\n"));
9100 		return (1);
9101 	}
9102 
9103 	cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
9104 	if (cur_version > cbp->cb_version) {
9105 		(void) printf(gettext("Pool '%s' is already formatted "
9106 		    "using more current version '%llu'.\n\n"),
9107 		    zpool_get_name(zhp), (u_longlong_t)cur_version);
9108 		return (0);
9109 	}
9110 
9111 	if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
9112 		(void) printf(gettext("Pool '%s' is already formatted "
9113 		    "using version %llu.\n\n"), zpool_get_name(zhp),
9114 		    (u_longlong_t)cbp->cb_version);
9115 		return (0);
9116 	}
9117 
9118 	if (cur_version != cbp->cb_version) {
9119 		modified_pool = B_TRUE;
9120 		ret = upgrade_version(zhp, cbp->cb_version);
9121 		if (ret != 0)
9122 			return (ret);
9123 	}
9124 
9125 	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
9126 		int count = 0;
9127 		ret = upgrade_enable_all(zhp, &count);
9128 		if (ret != 0)
9129 			return (ret);
9130 
9131 		if (count != 0) {
9132 			modified_pool = B_TRUE;
9133 		} else if (cur_version == SPA_VERSION) {
9134 			(void) printf(gettext("Pool '%s' already has all "
9135 			    "supported and requested features enabled.\n"),
9136 			    zpool_get_name(zhp));
9137 		}
9138 	}
9139 
9140 	if (modified_pool) {
9141 		(void) printf("\n");
9142 		(void) after_zpool_upgrade(zhp);
9143 	}
9144 
9145 	return (0);
9146 }
9147 
9148 /*
9149  * zpool upgrade
9150  * zpool upgrade -v
9151  * zpool upgrade [-V version] <-a | pool ...>
9152  *
9153  * With no arguments, display downrev'd ZFS pool available for upgrade.
9154  * Individual pools can be upgraded by specifying the pool, and '-a' will
9155  * upgrade all pools.
9156  */
9157 int
9158 zpool_do_upgrade(int argc, char **argv)
9159 {
9160 	int c;
9161 	upgrade_cbdata_t cb = { 0 };
9162 	int ret = 0;
9163 	boolean_t showversions = B_FALSE;
9164 	boolean_t upgradeall = B_FALSE;
9165 	char *end;
9166 
9167 
9168 	/* check options */
9169 	while ((c = getopt(argc, argv, ":avV:")) != -1) {
9170 		switch (c) {
9171 		case 'a':
9172 			upgradeall = B_TRUE;
9173 			break;
9174 		case 'v':
9175 			showversions = B_TRUE;
9176 			break;
9177 		case 'V':
9178 			cb.cb_version = strtoll(optarg, &end, 10);
9179 			if (*end != '\0' ||
9180 			    !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
9181 				(void) fprintf(stderr,
9182 				    gettext("invalid version '%s'\n"), optarg);
9183 				usage(B_FALSE);
9184 			}
9185 			break;
9186 		case ':':
9187 			(void) fprintf(stderr, gettext("missing argument for "
9188 			    "'%c' option\n"), optopt);
9189 			usage(B_FALSE);
9190 			break;
9191 		case '?':
9192 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
9193 			    optopt);
9194 			usage(B_FALSE);
9195 		}
9196 	}
9197 
9198 	cb.cb_argc = argc;
9199 	cb.cb_argv = argv;
9200 	argc -= optind;
9201 	argv += optind;
9202 
9203 	if (cb.cb_version == 0) {
9204 		cb.cb_version = SPA_VERSION;
9205 	} else if (!upgradeall && argc == 0) {
9206 		(void) fprintf(stderr, gettext("-V option is "
9207 		    "incompatible with other arguments\n"));
9208 		usage(B_FALSE);
9209 	}
9210 
9211 	if (showversions) {
9212 		if (upgradeall || argc != 0) {
9213 			(void) fprintf(stderr, gettext("-v option is "
9214 			    "incompatible with other arguments\n"));
9215 			usage(B_FALSE);
9216 		}
9217 	} else if (upgradeall) {
9218 		if (argc != 0) {
9219 			(void) fprintf(stderr, gettext("-a option should not "
9220 			    "be used along with a pool name\n"));
9221 			usage(B_FALSE);
9222 		}
9223 	}
9224 
9225 	(void) printf(gettext("This system supports ZFS pool feature "
9226 	    "flags.\n\n"));
9227 	if (showversions) {
9228 		int i;
9229 
9230 		(void) printf(gettext("The following features are "
9231 		    "supported:\n\n"));
9232 		(void) printf(gettext("FEAT DESCRIPTION\n"));
9233 		(void) printf("----------------------------------------------"
9234 		    "---------------\n");
9235 		for (i = 0; i < SPA_FEATURES; i++) {
9236 			zfeature_info_t *fi = &spa_feature_table[i];
9237 			if (!fi->fi_zfs_mod_supported)
9238 				continue;
9239 			const char *ro =
9240 			    (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
9241 			    " (read-only compatible)" : "";
9242 
9243 			(void) printf("%-37s%s\n", fi->fi_uname, ro);
9244 			(void) printf("     %s\n", fi->fi_desc);
9245 		}
9246 		(void) printf("\n");
9247 
9248 		(void) printf(gettext("The following legacy versions are also "
9249 		    "supported:\n\n"));
9250 		(void) printf(gettext("VER  DESCRIPTION\n"));
9251 		(void) printf("---  -----------------------------------------"
9252 		    "---------------\n");
9253 		(void) printf(gettext(" 1   Initial ZFS version\n"));
9254 		(void) printf(gettext(" 2   Ditto blocks "
9255 		    "(replicated metadata)\n"));
9256 		(void) printf(gettext(" 3   Hot spares and double parity "
9257 		    "RAID-Z\n"));
9258 		(void) printf(gettext(" 4   zpool history\n"));
9259 		(void) printf(gettext(" 5   Compression using the gzip "
9260 		    "algorithm\n"));
9261 		(void) printf(gettext(" 6   bootfs pool property\n"));
9262 		(void) printf(gettext(" 7   Separate intent log devices\n"));
9263 		(void) printf(gettext(" 8   Delegated administration\n"));
9264 		(void) printf(gettext(" 9   refquota and refreservation "
9265 		    "properties\n"));
9266 		(void) printf(gettext(" 10  Cache devices\n"));
9267 		(void) printf(gettext(" 11  Improved scrub performance\n"));
9268 		(void) printf(gettext(" 12  Snapshot properties\n"));
9269 		(void) printf(gettext(" 13  snapused property\n"));
9270 		(void) printf(gettext(" 14  passthrough-x aclinherit\n"));
9271 		(void) printf(gettext(" 15  user/group space accounting\n"));
9272 		(void) printf(gettext(" 16  stmf property support\n"));
9273 		(void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
9274 		(void) printf(gettext(" 18  Snapshot user holds\n"));
9275 		(void) printf(gettext(" 19  Log device removal\n"));
9276 		(void) printf(gettext(" 20  Compression using zle "
9277 		    "(zero-length encoding)\n"));
9278 		(void) printf(gettext(" 21  Deduplication\n"));
9279 		(void) printf(gettext(" 22  Received properties\n"));
9280 		(void) printf(gettext(" 23  Slim ZIL\n"));
9281 		(void) printf(gettext(" 24  System attributes\n"));
9282 		(void) printf(gettext(" 25  Improved scrub stats\n"));
9283 		(void) printf(gettext(" 26  Improved snapshot deletion "
9284 		    "performance\n"));
9285 		(void) printf(gettext(" 27  Improved snapshot creation "
9286 		    "performance\n"));
9287 		(void) printf(gettext(" 28  Multiple vdev replacements\n"));
9288 		(void) printf(gettext("\nFor more information on a particular "
9289 		    "version, including supported releases,\n"));
9290 		(void) printf(gettext("see the ZFS Administration Guide.\n\n"));
9291 	} else if (argc == 0 && upgradeall) {
9292 		cb.cb_first = B_TRUE;
9293 		ret = zpool_iter(g_zfs, upgrade_cb, &cb);
9294 		if (ret == 0 && cb.cb_first) {
9295 			if (cb.cb_version == SPA_VERSION) {
9296 				(void) printf(gettext("All pools are already "
9297 				    "formatted using feature flags.\n\n"));
9298 				(void) printf(gettext("Every feature flags "
9299 				    "pool already has all supported and "
9300 				    "requested features enabled.\n"));
9301 			} else {
9302 				(void) printf(gettext("All pools are already "
9303 				    "formatted with version %llu or higher.\n"),
9304 				    (u_longlong_t)cb.cb_version);
9305 			}
9306 		}
9307 	} else if (argc == 0) {
9308 		cb.cb_first = B_TRUE;
9309 		ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
9310 		assert(ret == 0);
9311 
9312 		if (cb.cb_first) {
9313 			(void) printf(gettext("All pools are formatted "
9314 			    "using feature flags.\n\n"));
9315 		} else {
9316 			(void) printf(gettext("\nUse 'zpool upgrade -v' "
9317 			    "for a list of available legacy versions.\n"));
9318 		}
9319 
9320 		cb.cb_first = B_TRUE;
9321 		ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
9322 		assert(ret == 0);
9323 
9324 		if (cb.cb_first) {
9325 			(void) printf(gettext("Every feature flags pool has "
9326 			    "all supported and requested features enabled.\n"));
9327 		} else {
9328 			(void) printf(gettext("\n"));
9329 		}
9330 	} else {
9331 		ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
9332 		    B_FALSE, upgrade_one, &cb);
9333 	}
9334 
9335 	return (ret);
9336 }
9337 
9338 typedef struct hist_cbdata {
9339 	boolean_t first;
9340 	boolean_t longfmt;
9341 	boolean_t internal;
9342 } hist_cbdata_t;
9343 
9344 static void
9345 print_history_records(nvlist_t *nvhis, hist_cbdata_t *cb)
9346 {
9347 	nvlist_t **records;
9348 	uint_t numrecords;
9349 	int i;
9350 
9351 	verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
9352 	    &records, &numrecords) == 0);
9353 	for (i = 0; i < numrecords; i++) {
9354 		nvlist_t *rec = records[i];
9355 		char tbuf[64] = "";
9356 
9357 		if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
9358 			time_t tsec;
9359 			struct tm t;
9360 
9361 			tsec = fnvlist_lookup_uint64(records[i],
9362 			    ZPOOL_HIST_TIME);
9363 			(void) localtime_r(&tsec, &t);
9364 			(void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
9365 		}
9366 
9367 		if (nvlist_exists(rec, ZPOOL_HIST_ELAPSED_NS)) {
9368 			uint64_t elapsed_ns = fnvlist_lookup_int64(records[i],
9369 			    ZPOOL_HIST_ELAPSED_NS);
9370 			(void) snprintf(tbuf + strlen(tbuf),
9371 			    sizeof (tbuf) - strlen(tbuf),
9372 			    " (%lldms)", (long long)elapsed_ns / 1000 / 1000);
9373 		}
9374 
9375 		if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
9376 			(void) printf("%s %s", tbuf,
9377 			    fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
9378 		} else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
9379 			int ievent =
9380 			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
9381 			if (!cb->internal)
9382 				continue;
9383 			if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
9384 				(void) printf("%s unrecognized record:\n",
9385 				    tbuf);
9386 				dump_nvlist(rec, 4);
9387 				continue;
9388 			}
9389 			(void) printf("%s [internal %s txg:%lld] %s", tbuf,
9390 			    zfs_history_event_names[ievent],
9391 			    (longlong_t)fnvlist_lookup_uint64(
9392 			    rec, ZPOOL_HIST_TXG),
9393 			    fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
9394 		} else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
9395 			if (!cb->internal)
9396 				continue;
9397 			(void) printf("%s [txg:%lld] %s", tbuf,
9398 			    (longlong_t)fnvlist_lookup_uint64(
9399 			    rec, ZPOOL_HIST_TXG),
9400 			    fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
9401 			if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
9402 				(void) printf(" %s (%llu)",
9403 				    fnvlist_lookup_string(rec,
9404 				    ZPOOL_HIST_DSNAME),
9405 				    (u_longlong_t)fnvlist_lookup_uint64(rec,
9406 				    ZPOOL_HIST_DSID));
9407 			}
9408 			(void) printf(" %s", fnvlist_lookup_string(rec,
9409 			    ZPOOL_HIST_INT_STR));
9410 		} else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
9411 			if (!cb->internal)
9412 				continue;
9413 			(void) printf("%s ioctl %s\n", tbuf,
9414 			    fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
9415 			if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
9416 				(void) printf("    input:\n");
9417 				dump_nvlist(fnvlist_lookup_nvlist(rec,
9418 				    ZPOOL_HIST_INPUT_NVL), 8);
9419 			}
9420 			if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
9421 				(void) printf("    output:\n");
9422 				dump_nvlist(fnvlist_lookup_nvlist(rec,
9423 				    ZPOOL_HIST_OUTPUT_NVL), 8);
9424 			}
9425 			if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_SIZE)) {
9426 				(void) printf("    output nvlist omitted; "
9427 				    "original size: %lldKB\n",
9428 				    (longlong_t)fnvlist_lookup_int64(rec,
9429 				    ZPOOL_HIST_OUTPUT_SIZE) / 1024);
9430 			}
9431 			if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
9432 				(void) printf("    errno: %lld\n",
9433 				    (longlong_t)fnvlist_lookup_int64(rec,
9434 				    ZPOOL_HIST_ERRNO));
9435 			}
9436 		} else {
9437 			if (!cb->internal)
9438 				continue;
9439 			(void) printf("%s unrecognized record:\n", tbuf);
9440 			dump_nvlist(rec, 4);
9441 		}
9442 
9443 		if (!cb->longfmt) {
9444 			(void) printf("\n");
9445 			continue;
9446 		}
9447 		(void) printf(" [");
9448 		if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
9449 			uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
9450 			struct passwd *pwd = getpwuid(who);
9451 			(void) printf("user %d ", (int)who);
9452 			if (pwd != NULL)
9453 				(void) printf("(%s) ", pwd->pw_name);
9454 		}
9455 		if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
9456 			(void) printf("on %s",
9457 			    fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
9458 		}
9459 		if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
9460 			(void) printf(":%s",
9461 			    fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
9462 		}
9463 
9464 		(void) printf("]");
9465 		(void) printf("\n");
9466 	}
9467 }
9468 
9469 /*
9470  * Print out the command history for a specific pool.
9471  */
9472 static int
9473 get_history_one(zpool_handle_t *zhp, void *data)
9474 {
9475 	nvlist_t *nvhis;
9476 	int ret;
9477 	hist_cbdata_t *cb = (hist_cbdata_t *)data;
9478 	uint64_t off = 0;
9479 	boolean_t eof = B_FALSE;
9480 
9481 	cb->first = B_FALSE;
9482 
9483 	(void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
9484 
9485 	while (!eof) {
9486 		if ((ret = zpool_get_history(zhp, &nvhis, &off, &eof)) != 0)
9487 			return (ret);
9488 
9489 		print_history_records(nvhis, cb);
9490 		nvlist_free(nvhis);
9491 	}
9492 	(void) printf("\n");
9493 
9494 	return (ret);
9495 }
9496 
9497 /*
9498  * zpool history <pool>
9499  *
9500  * Displays the history of commands that modified pools.
9501  */
9502 int
9503 zpool_do_history(int argc, char **argv)
9504 {
9505 	hist_cbdata_t cbdata = { 0 };
9506 	int ret;
9507 	int c;
9508 
9509 	cbdata.first = B_TRUE;
9510 	/* check options */
9511 	while ((c = getopt(argc, argv, "li")) != -1) {
9512 		switch (c) {
9513 		case 'l':
9514 			cbdata.longfmt = B_TRUE;
9515 			break;
9516 		case 'i':
9517 			cbdata.internal = B_TRUE;
9518 			break;
9519 		case '?':
9520 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
9521 			    optopt);
9522 			usage(B_FALSE);
9523 		}
9524 	}
9525 	argc -= optind;
9526 	argv += optind;
9527 
9528 	ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
9529 	    B_FALSE, get_history_one, &cbdata);
9530 
9531 	if (argc == 0 && cbdata.first == B_TRUE) {
9532 		(void) fprintf(stderr, gettext("no pools available\n"));
9533 		return (0);
9534 	}
9535 
9536 	return (ret);
9537 }
9538 
9539 typedef struct ev_opts {
9540 	int verbose;
9541 	int scripted;
9542 	int follow;
9543 	int clear;
9544 	char poolname[ZFS_MAX_DATASET_NAME_LEN];
9545 } ev_opts_t;
9546 
9547 static void
9548 zpool_do_events_short(nvlist_t *nvl, ev_opts_t *opts)
9549 {
9550 	char ctime_str[26], str[32], *ptr;
9551 	int64_t *tv;
9552 	uint_t n;
9553 
9554 	verify(nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0);
9555 	memset(str, ' ', 32);
9556 	(void) ctime_r((const time_t *)&tv[0], ctime_str);
9557 	(void) memcpy(str, ctime_str+4,  6);		/* 'Jun 30' */
9558 	(void) memcpy(str+7, ctime_str+20, 4);		/* '1993' */
9559 	(void) memcpy(str+12, ctime_str+11, 8);		/* '21:49:08' */
9560 	(void) sprintf(str+20, ".%09lld", (longlong_t)tv[1]); /* '.123456789' */
9561 	if (opts->scripted)
9562 		(void) printf(gettext("%s\t"), str);
9563 	else
9564 		(void) printf(gettext("%s "), str);
9565 
9566 	verify(nvlist_lookup_string(nvl, FM_CLASS, &ptr) == 0);
9567 	(void) printf(gettext("%s\n"), ptr);
9568 }
9569 
9570 static void
9571 zpool_do_events_nvprint(nvlist_t *nvl, int depth)
9572 {
9573 	nvpair_t *nvp;
9574 
9575 	for (nvp = nvlist_next_nvpair(nvl, NULL);
9576 	    nvp != NULL; nvp = nvlist_next_nvpair(nvl, nvp)) {
9577 
9578 		data_type_t type = nvpair_type(nvp);
9579 		const char *name = nvpair_name(nvp);
9580 
9581 		boolean_t b;
9582 		uint8_t i8;
9583 		uint16_t i16;
9584 		uint32_t i32;
9585 		uint64_t i64;
9586 		char *str;
9587 		nvlist_t *cnv;
9588 
9589 		printf(gettext("%*s%s = "), depth, "", name);
9590 
9591 		switch (type) {
9592 		case DATA_TYPE_BOOLEAN:
9593 			printf(gettext("%s"), "1");
9594 			break;
9595 
9596 		case DATA_TYPE_BOOLEAN_VALUE:
9597 			(void) nvpair_value_boolean_value(nvp, &b);
9598 			printf(gettext("%s"), b ? "1" : "0");
9599 			break;
9600 
9601 		case DATA_TYPE_BYTE:
9602 			(void) nvpair_value_byte(nvp, &i8);
9603 			printf(gettext("0x%x"), i8);
9604 			break;
9605 
9606 		case DATA_TYPE_INT8:
9607 			(void) nvpair_value_int8(nvp, (void *)&i8);
9608 			printf(gettext("0x%x"), i8);
9609 			break;
9610 
9611 		case DATA_TYPE_UINT8:
9612 			(void) nvpair_value_uint8(nvp, &i8);
9613 			printf(gettext("0x%x"), i8);
9614 			break;
9615 
9616 		case DATA_TYPE_INT16:
9617 			(void) nvpair_value_int16(nvp, (void *)&i16);
9618 			printf(gettext("0x%x"), i16);
9619 			break;
9620 
9621 		case DATA_TYPE_UINT16:
9622 			(void) nvpair_value_uint16(nvp, &i16);
9623 			printf(gettext("0x%x"), i16);
9624 			break;
9625 
9626 		case DATA_TYPE_INT32:
9627 			(void) nvpair_value_int32(nvp, (void *)&i32);
9628 			printf(gettext("0x%x"), i32);
9629 			break;
9630 
9631 		case DATA_TYPE_UINT32:
9632 			(void) nvpair_value_uint32(nvp, &i32);
9633 			printf(gettext("0x%x"), i32);
9634 			break;
9635 
9636 		case DATA_TYPE_INT64:
9637 			(void) nvpair_value_int64(nvp, (void *)&i64);
9638 			printf(gettext("0x%llx"), (u_longlong_t)i64);
9639 			break;
9640 
9641 		case DATA_TYPE_UINT64:
9642 			(void) nvpair_value_uint64(nvp, &i64);
9643 			/*
9644 			 * translate vdev state values to readable
9645 			 * strings to aide zpool events consumers
9646 			 */
9647 			if (strcmp(name,
9648 			    FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE) == 0 ||
9649 			    strcmp(name,
9650 			    FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE) == 0) {
9651 				printf(gettext("\"%s\" (0x%llx)"),
9652 				    zpool_state_to_name(i64, VDEV_AUX_NONE),
9653 				    (u_longlong_t)i64);
9654 			} else {
9655 				printf(gettext("0x%llx"), (u_longlong_t)i64);
9656 			}
9657 			break;
9658 
9659 		case DATA_TYPE_HRTIME:
9660 			(void) nvpair_value_hrtime(nvp, (void *)&i64);
9661 			printf(gettext("0x%llx"), (u_longlong_t)i64);
9662 			break;
9663 
9664 		case DATA_TYPE_STRING:
9665 			(void) nvpair_value_string(nvp, &str);
9666 			printf(gettext("\"%s\""), str ? str : "<NULL>");
9667 			break;
9668 
9669 		case DATA_TYPE_NVLIST:
9670 			printf(gettext("(embedded nvlist)\n"));
9671 			(void) nvpair_value_nvlist(nvp, &cnv);
9672 			zpool_do_events_nvprint(cnv, depth + 8);
9673 			printf(gettext("%*s(end %s)"), depth, "", name);
9674 			break;
9675 
9676 		case DATA_TYPE_NVLIST_ARRAY: {
9677 			nvlist_t **val;
9678 			uint_t i, nelem;
9679 
9680 			(void) nvpair_value_nvlist_array(nvp, &val, &nelem);
9681 			printf(gettext("(%d embedded nvlists)\n"), nelem);
9682 			for (i = 0; i < nelem; i++) {
9683 				printf(gettext("%*s%s[%d] = %s\n"),
9684 				    depth, "", name, i, "(embedded nvlist)");
9685 				zpool_do_events_nvprint(val[i], depth + 8);
9686 				printf(gettext("%*s(end %s[%i])\n"),
9687 				    depth, "", name, i);
9688 			}
9689 			printf(gettext("%*s(end %s)\n"), depth, "", name);
9690 			}
9691 			break;
9692 
9693 		case DATA_TYPE_INT8_ARRAY: {
9694 			int8_t *val;
9695 			uint_t i, nelem;
9696 
9697 			(void) nvpair_value_int8_array(nvp, &val, &nelem);
9698 			for (i = 0; i < nelem; i++)
9699 				printf(gettext("0x%x "), val[i]);
9700 
9701 			break;
9702 			}
9703 
9704 		case DATA_TYPE_UINT8_ARRAY: {
9705 			uint8_t *val;
9706 			uint_t i, nelem;
9707 
9708 			(void) nvpair_value_uint8_array(nvp, &val, &nelem);
9709 			for (i = 0; i < nelem; i++)
9710 				printf(gettext("0x%x "), val[i]);
9711 
9712 			break;
9713 			}
9714 
9715 		case DATA_TYPE_INT16_ARRAY: {
9716 			int16_t *val;
9717 			uint_t i, nelem;
9718 
9719 			(void) nvpair_value_int16_array(nvp, &val, &nelem);
9720 			for (i = 0; i < nelem; i++)
9721 				printf(gettext("0x%x "), val[i]);
9722 
9723 			break;
9724 			}
9725 
9726 		case DATA_TYPE_UINT16_ARRAY: {
9727 			uint16_t *val;
9728 			uint_t i, nelem;
9729 
9730 			(void) nvpair_value_uint16_array(nvp, &val, &nelem);
9731 			for (i = 0; i < nelem; i++)
9732 				printf(gettext("0x%x "), val[i]);
9733 
9734 			break;
9735 			}
9736 
9737 		case DATA_TYPE_INT32_ARRAY: {
9738 			int32_t *val;
9739 			uint_t i, nelem;
9740 
9741 			(void) nvpair_value_int32_array(nvp, &val, &nelem);
9742 			for (i = 0; i < nelem; i++)
9743 				printf(gettext("0x%x "), val[i]);
9744 
9745 			break;
9746 			}
9747 
9748 		case DATA_TYPE_UINT32_ARRAY: {
9749 			uint32_t *val;
9750 			uint_t i, nelem;
9751 
9752 			(void) nvpair_value_uint32_array(nvp, &val, &nelem);
9753 			for (i = 0; i < nelem; i++)
9754 				printf(gettext("0x%x "), val[i]);
9755 
9756 			break;
9757 			}
9758 
9759 		case DATA_TYPE_INT64_ARRAY: {
9760 			int64_t *val;
9761 			uint_t i, nelem;
9762 
9763 			(void) nvpair_value_int64_array(nvp, &val, &nelem);
9764 			for (i = 0; i < nelem; i++)
9765 				printf(gettext("0x%llx "),
9766 				    (u_longlong_t)val[i]);
9767 
9768 			break;
9769 			}
9770 
9771 		case DATA_TYPE_UINT64_ARRAY: {
9772 			uint64_t *val;
9773 			uint_t i, nelem;
9774 
9775 			(void) nvpair_value_uint64_array(nvp, &val, &nelem);
9776 			for (i = 0; i < nelem; i++)
9777 				printf(gettext("0x%llx "),
9778 				    (u_longlong_t)val[i]);
9779 
9780 			break;
9781 			}
9782 
9783 		case DATA_TYPE_STRING_ARRAY: {
9784 			char **str;
9785 			uint_t i, nelem;
9786 
9787 			(void) nvpair_value_string_array(nvp, &str, &nelem);
9788 			for (i = 0; i < nelem; i++)
9789 				printf(gettext("\"%s\" "),
9790 				    str[i] ? str[i] : "<NULL>");
9791 
9792 			break;
9793 			}
9794 
9795 		case DATA_TYPE_BOOLEAN_ARRAY:
9796 		case DATA_TYPE_BYTE_ARRAY:
9797 		case DATA_TYPE_DOUBLE:
9798 		case DATA_TYPE_DONTCARE:
9799 		case DATA_TYPE_UNKNOWN:
9800 			printf(gettext("<unknown>"));
9801 			break;
9802 		}
9803 
9804 		printf(gettext("\n"));
9805 	}
9806 }
9807 
9808 static int
9809 zpool_do_events_next(ev_opts_t *opts)
9810 {
9811 	nvlist_t *nvl;
9812 	int zevent_fd, ret, dropped;
9813 	char *pool;
9814 
9815 	zevent_fd = open(ZFS_DEV, O_RDWR);
9816 	VERIFY(zevent_fd >= 0);
9817 
9818 	if (!opts->scripted)
9819 		(void) printf(gettext("%-30s %s\n"), "TIME", "CLASS");
9820 
9821 	while (1) {
9822 		ret = zpool_events_next(g_zfs, &nvl, &dropped,
9823 		    (opts->follow ? ZEVENT_NONE : ZEVENT_NONBLOCK), zevent_fd);
9824 		if (ret || nvl == NULL)
9825 			break;
9826 
9827 		if (dropped > 0)
9828 			(void) printf(gettext("dropped %d events\n"), dropped);
9829 
9830 		if (strlen(opts->poolname) > 0 &&
9831 		    nvlist_lookup_string(nvl, FM_FMRI_ZFS_POOL, &pool) == 0 &&
9832 		    strcmp(opts->poolname, pool) != 0)
9833 			continue;
9834 
9835 		zpool_do_events_short(nvl, opts);
9836 
9837 		if (opts->verbose) {
9838 			zpool_do_events_nvprint(nvl, 8);
9839 			printf(gettext("\n"));
9840 		}
9841 		(void) fflush(stdout);
9842 
9843 		nvlist_free(nvl);
9844 	}
9845 
9846 	VERIFY(0 == close(zevent_fd));
9847 
9848 	return (ret);
9849 }
9850 
9851 static int
9852 zpool_do_events_clear(ev_opts_t *opts)
9853 {
9854 	int count, ret;
9855 
9856 	ret = zpool_events_clear(g_zfs, &count);
9857 	if (!ret)
9858 		(void) printf(gettext("cleared %d events\n"), count);
9859 
9860 	return (ret);
9861 }
9862 
9863 /*
9864  * zpool events [-vHf [pool] | -c]
9865  *
9866  * Displays events logs by ZFS.
9867  */
9868 int
9869 zpool_do_events(int argc, char **argv)
9870 {
9871 	ev_opts_t opts = { 0 };
9872 	int ret;
9873 	int c;
9874 
9875 	/* check options */
9876 	while ((c = getopt(argc, argv, "vHfc")) != -1) {
9877 		switch (c) {
9878 		case 'v':
9879 			opts.verbose = 1;
9880 			break;
9881 		case 'H':
9882 			opts.scripted = 1;
9883 			break;
9884 		case 'f':
9885 			opts.follow = 1;
9886 			break;
9887 		case 'c':
9888 			opts.clear = 1;
9889 			break;
9890 		case '?':
9891 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
9892 			    optopt);
9893 			usage(B_FALSE);
9894 		}
9895 	}
9896 	argc -= optind;
9897 	argv += optind;
9898 
9899 	if (argc > 1) {
9900 		(void) fprintf(stderr, gettext("too many arguments\n"));
9901 		usage(B_FALSE);
9902 	} else if (argc == 1) {
9903 		(void) strlcpy(opts.poolname, argv[0], sizeof (opts.poolname));
9904 		if (!zfs_name_valid(opts.poolname, ZFS_TYPE_POOL)) {
9905 			(void) fprintf(stderr,
9906 			    gettext("invalid pool name '%s'\n"), opts.poolname);
9907 			usage(B_FALSE);
9908 		}
9909 	}
9910 
9911 	if ((argc == 1 || opts.verbose || opts.scripted || opts.follow) &&
9912 	    opts.clear) {
9913 		(void) fprintf(stderr,
9914 		    gettext("invalid options combined with -c\n"));
9915 		usage(B_FALSE);
9916 	}
9917 
9918 	if (opts.clear)
9919 		ret = zpool_do_events_clear(&opts);
9920 	else
9921 		ret = zpool_do_events_next(&opts);
9922 
9923 	return (ret);
9924 }
9925 
9926 static int
9927 get_callback_vdev(zpool_handle_t *zhp, char *vdevname, void *data)
9928 {
9929 	zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
9930 	char value[ZFS_MAXPROPLEN];
9931 	zprop_source_t srctype;
9932 
9933 	for (zprop_list_t *pl = cbp->cb_proplist; pl != NULL;
9934 	    pl = pl->pl_next) {
9935 		char *prop_name;
9936 		/*
9937 		 * If the first property is pool name, it is a special
9938 		 * placeholder that we can skip. This will also skip
9939 		 * over the name property when 'all' is specified.
9940 		 */
9941 		if (pl->pl_prop == ZPOOL_PROP_NAME &&
9942 		    pl == cbp->cb_proplist)
9943 			continue;
9944 
9945 		if (pl->pl_prop == ZPROP_INVAL) {
9946 			prop_name = pl->pl_user_prop;
9947 		} else {
9948 			prop_name = (char *)vdev_prop_to_name(pl->pl_prop);
9949 		}
9950 		if (zpool_get_vdev_prop(zhp, vdevname, pl->pl_prop,
9951 		    prop_name, value, sizeof (value), &srctype,
9952 		    cbp->cb_literal) == 0) {
9953 			zprop_print_one_property(vdevname, cbp, prop_name,
9954 			    value, srctype, NULL, NULL);
9955 		}
9956 	}
9957 
9958 	return (0);
9959 }
9960 
9961 static int
9962 get_callback_vdev_width_cb(void *zhp_data, nvlist_t *nv, void *data)
9963 {
9964 	zpool_handle_t *zhp = zhp_data;
9965 	zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
9966 	char *vdevname = zpool_vdev_name(g_zfs, zhp, nv,
9967 	    cbp->cb_vdevs.cb_name_flags);
9968 	int ret;
9969 
9970 	/* Adjust the column widths for the vdev properties */
9971 	ret = vdev_expand_proplist(zhp, vdevname, &cbp->cb_proplist);
9972 
9973 	return (ret);
9974 }
9975 
9976 static int
9977 get_callback_vdev_cb(void *zhp_data, nvlist_t *nv, void *data)
9978 {
9979 	zpool_handle_t *zhp = zhp_data;
9980 	zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
9981 	char *vdevname = zpool_vdev_name(g_zfs, zhp, nv,
9982 	    cbp->cb_vdevs.cb_name_flags);
9983 	int ret;
9984 
9985 	/* Display the properties */
9986 	ret = get_callback_vdev(zhp, vdevname, data);
9987 
9988 	return (ret);
9989 }
9990 
9991 static int
9992 get_callback(zpool_handle_t *zhp, void *data)
9993 {
9994 	zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
9995 	char value[MAXNAMELEN];
9996 	zprop_source_t srctype;
9997 	zprop_list_t *pl;
9998 	int vid;
9999 
10000 	if (cbp->cb_type == ZFS_TYPE_VDEV) {
10001 		if (strcmp(cbp->cb_vdevs.cb_names[0], "all-vdevs") == 0) {
10002 			for_each_vdev(zhp, get_callback_vdev_width_cb, data);
10003 			for_each_vdev(zhp, get_callback_vdev_cb, data);
10004 		} else {
10005 			/* Adjust column widths for vdev properties */
10006 			for (vid = 0; vid < cbp->cb_vdevs.cb_names_count;
10007 			    vid++) {
10008 				vdev_expand_proplist(zhp,
10009 				    cbp->cb_vdevs.cb_names[vid],
10010 				    &cbp->cb_proplist);
10011 			}
10012 			/* Display the properties */
10013 			for (vid = 0; vid < cbp->cb_vdevs.cb_names_count;
10014 			    vid++) {
10015 				get_callback_vdev(zhp,
10016 				    cbp->cb_vdevs.cb_names[vid], data);
10017 			}
10018 		}
10019 	} else {
10020 		assert(cbp->cb_type == ZFS_TYPE_POOL);
10021 		for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
10022 			/*
10023 			 * Skip the special fake placeholder. This will also
10024 			 * skip over the name property when 'all' is specified.
10025 			 */
10026 			if (pl->pl_prop == ZPOOL_PROP_NAME &&
10027 			    pl == cbp->cb_proplist)
10028 				continue;
10029 
10030 			if (pl->pl_prop == ZPROP_INVAL &&
10031 			    (zpool_prop_feature(pl->pl_user_prop) ||
10032 			    zpool_prop_unsupported(pl->pl_user_prop))) {
10033 				srctype = ZPROP_SRC_LOCAL;
10034 
10035 				if (zpool_prop_get_feature(zhp,
10036 				    pl->pl_user_prop, value,
10037 				    sizeof (value)) == 0) {
10038 					zprop_print_one_property(
10039 					    zpool_get_name(zhp), cbp,
10040 					    pl->pl_user_prop, value, srctype,
10041 					    NULL, NULL);
10042 				}
10043 			} else {
10044 				if (zpool_get_prop(zhp, pl->pl_prop, value,
10045 				    sizeof (value), &srctype,
10046 				    cbp->cb_literal) != 0)
10047 					continue;
10048 
10049 				zprop_print_one_property(zpool_get_name(zhp),
10050 				    cbp, zpool_prop_to_name(pl->pl_prop),
10051 				    value, srctype, NULL, NULL);
10052 			}
10053 		}
10054 	}
10055 
10056 	return (0);
10057 }
10058 
10059 /*
10060  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
10061  *
10062  *	-H	Scripted mode.  Don't display headers, and separate properties
10063  *		by a single tab.
10064  *	-o	List of columns to display.  Defaults to
10065  *		"name,property,value,source".
10066  * 	-p	Display values in parsable (exact) format.
10067  *
10068  * Get properties of pools in the system. Output space statistics
10069  * for each one as well as other attributes.
10070  */
10071 int
10072 zpool_do_get(int argc, char **argv)
10073 {
10074 	zprop_get_cbdata_t cb = { 0 };
10075 	zprop_list_t fake_name = { 0 };
10076 	int ret;
10077 	int c, i;
10078 	char *value;
10079 	char *propstr = NULL;
10080 
10081 	cb.cb_first = B_TRUE;
10082 
10083 	/*
10084 	 * Set up default columns and sources.
10085 	 */
10086 	cb.cb_sources = ZPROP_SRC_ALL;
10087 	cb.cb_columns[0] = GET_COL_NAME;
10088 	cb.cb_columns[1] = GET_COL_PROPERTY;
10089 	cb.cb_columns[2] = GET_COL_VALUE;
10090 	cb.cb_columns[3] = GET_COL_SOURCE;
10091 	cb.cb_type = ZFS_TYPE_POOL;
10092 	cb.cb_vdevs.cb_name_flags |= VDEV_NAME_TYPE_ID;
10093 	current_prop_type = cb.cb_type;
10094 
10095 	/* check options */
10096 	while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
10097 		switch (c) {
10098 		case 'p':
10099 			cb.cb_literal = B_TRUE;
10100 			break;
10101 		case 'H':
10102 			cb.cb_scripted = B_TRUE;
10103 			break;
10104 		case 'o':
10105 			bzero(&cb.cb_columns, sizeof (cb.cb_columns));
10106 			i = 0;
10107 			while (*optarg != '\0') {
10108 				static char *col_subopts[] =
10109 				{ "name", "property", "value", "source",
10110 				"all", NULL };
10111 
10112 				if (i == ZFS_GET_NCOLS) {
10113 					(void) fprintf(stderr, gettext("too "
10114 					"many fields given to -o "
10115 					"option\n"));
10116 					usage(B_FALSE);
10117 				}
10118 
10119 				switch (getsubopt(&optarg, col_subopts,
10120 				    &value)) {
10121 				case 0:
10122 					cb.cb_columns[i++] = GET_COL_NAME;
10123 					break;
10124 				case 1:
10125 					cb.cb_columns[i++] = GET_COL_PROPERTY;
10126 					break;
10127 				case 2:
10128 					cb.cb_columns[i++] = GET_COL_VALUE;
10129 					break;
10130 				case 3:
10131 					cb.cb_columns[i++] = GET_COL_SOURCE;
10132 					break;
10133 				case 4:
10134 					if (i > 0) {
10135 						(void) fprintf(stderr,
10136 						    gettext("\"all\" conflicts "
10137 						    "with specific fields "
10138 						    "given to -o option\n"));
10139 						usage(B_FALSE);
10140 					}
10141 					cb.cb_columns[0] = GET_COL_NAME;
10142 					cb.cb_columns[1] = GET_COL_PROPERTY;
10143 					cb.cb_columns[2] = GET_COL_VALUE;
10144 					cb.cb_columns[3] = GET_COL_SOURCE;
10145 					i = ZFS_GET_NCOLS;
10146 					break;
10147 				default:
10148 					(void) fprintf(stderr,
10149 					    gettext("invalid column name "
10150 					    "'%s'\n"), value);
10151 					usage(B_FALSE);
10152 				}
10153 			}
10154 			break;
10155 		case '?':
10156 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
10157 			    optopt);
10158 			usage(B_FALSE);
10159 		}
10160 	}
10161 
10162 	argc -= optind;
10163 	argv += optind;
10164 
10165 	if (argc < 1) {
10166 		(void) fprintf(stderr, gettext("missing property "
10167 		    "argument\n"));
10168 		usage(B_FALSE);
10169 	}
10170 
10171 	/* Properties list is needed later by zprop_get_list() */
10172 	propstr = argv[0];
10173 
10174 	argc--;
10175 	argv++;
10176 
10177 	if (argc == 0) {
10178 		/* No args, so just print the defaults. */
10179 	} else if (are_all_pools(argc, argv)) {
10180 		/* All the args are pool names */
10181 	} else if (are_all_pools(1, argv)) {
10182 		/* The first arg is a pool name */
10183 		if ((argc == 2 && strcmp(argv[1], "all-vdevs") == 0) ||
10184 		    are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
10185 		    &cb.cb_vdevs)) {
10186 			/* ... and the rest are vdev names */
10187 			cb.cb_vdevs.cb_names = argv + 1;
10188 			cb.cb_vdevs.cb_names_count = argc - 1;
10189 			cb.cb_type = ZFS_TYPE_VDEV;
10190 			argc = 1; /* One pool to process */
10191 		} else {
10192 			fprintf(stderr, gettext("Expected a list of vdevs in"
10193 			    " \"%s\", but got:\n"), argv[0]);
10194 			error_list_unresolved_vdevs(argc - 1, argv + 1,
10195 			    argv[0], &cb.cb_vdevs);
10196 			fprintf(stderr, "\n");
10197 			usage(B_FALSE);
10198 			return (1);
10199 		}
10200 	} else {
10201 		/*
10202 		 * The first arg isn't a pool name,
10203 		 */
10204 		fprintf(stderr, gettext("missing pool name.\n"));
10205 		fprintf(stderr, "\n");
10206 		usage(B_FALSE);
10207 		return (1);
10208 	}
10209 
10210 	if (zprop_get_list(g_zfs, propstr, &cb.cb_proplist,
10211 	    cb.cb_type) != 0) {
10212 		/* Use correct list of valid properties (pool or vdev) */
10213 		current_prop_type = cb.cb_type;
10214 		usage(B_FALSE);
10215 	}
10216 
10217 	if (cb.cb_proplist != NULL) {
10218 		fake_name.pl_prop = ZPOOL_PROP_NAME;
10219 		fake_name.pl_width = strlen(gettext("NAME"));
10220 		fake_name.pl_next = cb.cb_proplist;
10221 		cb.cb_proplist = &fake_name;
10222 	}
10223 
10224 	ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist, cb.cb_type,
10225 	    cb.cb_literal, get_callback, &cb);
10226 
10227 	if (cb.cb_proplist == &fake_name)
10228 		zprop_free_list(fake_name.pl_next);
10229 	else
10230 		zprop_free_list(cb.cb_proplist);
10231 
10232 	return (ret);
10233 }
10234 
10235 typedef struct set_cbdata {
10236 	char *cb_propname;
10237 	char *cb_value;
10238 	zfs_type_t cb_type;
10239 	vdev_cbdata_t cb_vdevs;
10240 	boolean_t cb_any_successful;
10241 } set_cbdata_t;
10242 
10243 static int
10244 set_pool_callback(zpool_handle_t *zhp, set_cbdata_t *cb)
10245 {
10246 	int error;
10247 
10248 	/* Check if we have out-of-bounds features */
10249 	if (strcmp(cb->cb_propname, ZPOOL_CONFIG_COMPATIBILITY) == 0) {
10250 		boolean_t features[SPA_FEATURES];
10251 		if (zpool_do_load_compat(cb->cb_value, features) !=
10252 		    ZPOOL_COMPATIBILITY_OK)
10253 			return (-1);
10254 
10255 		nvlist_t *enabled = zpool_get_features(zhp);
10256 		spa_feature_t i;
10257 		for (i = 0; i < SPA_FEATURES; i++) {
10258 			const char *fguid = spa_feature_table[i].fi_guid;
10259 			if (nvlist_exists(enabled, fguid) && !features[i])
10260 				break;
10261 		}
10262 		if (i < SPA_FEATURES)
10263 			(void) fprintf(stderr, gettext("Warning: one or "
10264 			    "more features already enabled on pool '%s'\n"
10265 			    "are not present in this compatibility set.\n"),
10266 			    zpool_get_name(zhp));
10267 	}
10268 
10269 	/* if we're setting a feature, check it's in compatibility set */
10270 	if (zpool_prop_feature(cb->cb_propname) &&
10271 	    strcmp(cb->cb_value, ZFS_FEATURE_ENABLED) == 0) {
10272 		char *fname = strchr(cb->cb_propname, '@') + 1;
10273 		spa_feature_t f;
10274 
10275 		if (zfeature_lookup_name(fname, &f) == 0) {
10276 			char compat[ZFS_MAXPROPLEN];
10277 			if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY,
10278 			    compat, ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
10279 				compat[0] = '\0';
10280 
10281 			boolean_t features[SPA_FEATURES];
10282 			if (zpool_do_load_compat(compat, features) !=
10283 			    ZPOOL_COMPATIBILITY_OK) {
10284 				(void) fprintf(stderr, gettext("Error: "
10285 				    "cannot enable feature '%s' on pool '%s'\n"
10286 				    "because the pool's 'compatibility' "
10287 				    "property cannot be parsed.\n"),
10288 				    fname, zpool_get_name(zhp));
10289 				return (-1);
10290 			}
10291 
10292 			if (!features[f]) {
10293 				(void) fprintf(stderr, gettext("Error: "
10294 				    "cannot enable feature '%s' on pool '%s'\n"
10295 				    "as it is not specified in this pool's "
10296 				    "current compatibility set.\n"
10297 				    "Consider setting 'compatibility' to a "
10298 				    "less restrictive set, or to 'off'.\n"),
10299 				    fname, zpool_get_name(zhp));
10300 				return (-1);
10301 			}
10302 		}
10303 	}
10304 
10305 	error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
10306 
10307 	return (error);
10308 }
10309 
10310 static int
10311 set_callback(zpool_handle_t *zhp, void *data)
10312 {
10313 	int error;
10314 	set_cbdata_t *cb = (set_cbdata_t *)data;
10315 
10316 	if (cb->cb_type == ZFS_TYPE_VDEV) {
10317 		error = zpool_set_vdev_prop(zhp, *cb->cb_vdevs.cb_names,
10318 		    cb->cb_propname, cb->cb_value);
10319 	} else {
10320 		assert(cb->cb_type == ZFS_TYPE_POOL);
10321 		error = set_pool_callback(zhp, cb);
10322 	}
10323 
10324 	cb->cb_any_successful = !error;
10325 	return (error);
10326 }
10327 
10328 int
10329 zpool_do_set(int argc, char **argv)
10330 {
10331 	set_cbdata_t cb = { 0 };
10332 	int error;
10333 
10334 	current_prop_type = ZFS_TYPE_POOL;
10335 	if (argc > 1 && argv[1][0] == '-') {
10336 		(void) fprintf(stderr, gettext("invalid option '%c'\n"),
10337 		    argv[1][1]);
10338 		usage(B_FALSE);
10339 	}
10340 
10341 	if (argc < 2) {
10342 		(void) fprintf(stderr, gettext("missing property=value "
10343 		    "argument\n"));
10344 		usage(B_FALSE);
10345 	}
10346 
10347 	if (argc < 3) {
10348 		(void) fprintf(stderr, gettext("missing pool name\n"));
10349 		usage(B_FALSE);
10350 	}
10351 
10352 	if (argc > 4) {
10353 		(void) fprintf(stderr, gettext("too many pool names\n"));
10354 		usage(B_FALSE);
10355 	}
10356 
10357 	cb.cb_propname = argv[1];
10358 	cb.cb_type = ZFS_TYPE_POOL;
10359 	cb.cb_vdevs.cb_name_flags |= VDEV_NAME_TYPE_ID;
10360 	cb.cb_value = strchr(cb.cb_propname, '=');
10361 	if (cb.cb_value == NULL) {
10362 		(void) fprintf(stderr, gettext("missing value in "
10363 		    "property=value argument\n"));
10364 		usage(B_FALSE);
10365 	}
10366 
10367 	*(cb.cb_value) = '\0';
10368 	cb.cb_value++;
10369 	argc -= 2;
10370 	argv += 2;
10371 
10372 	if (are_vdevs_in_pool(argc, argv, NULL, &cb.cb_vdevs)) {
10373 		/* Argument is a vdev */
10374 		cb.cb_vdevs.cb_names = argv;
10375 		cb.cb_vdevs.cb_names_count = 1;
10376 		cb.cb_type = ZFS_TYPE_VDEV;
10377 		argc = 0; /* No pools to process */
10378 	} else if (are_all_pools(1, argv)) {
10379 		/* The first arg is a pool name */
10380 		if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
10381 		    &cb.cb_vdevs)) {
10382 			/* 2nd argument is a vdev */
10383 			cb.cb_vdevs.cb_names = argv + 1;
10384 			cb.cb_vdevs.cb_names_count = 1;
10385 			cb.cb_type = ZFS_TYPE_VDEV;
10386 			argc = 1; /* One pool to process */
10387 		} else if (argc > 1) {
10388 			(void) fprintf(stderr,
10389 			    gettext("too many pool names\n"));
10390 			usage(B_FALSE);
10391 		}
10392 	}
10393 
10394 	error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
10395 	    B_FALSE, set_callback, &cb);
10396 
10397 	return (error);
10398 }
10399 
10400 /* Add up the total number of bytes left to initialize/trim across all vdevs */
10401 static uint64_t
10402 vdev_activity_remaining(nvlist_t *nv, zpool_wait_activity_t activity)
10403 {
10404 	uint64_t bytes_remaining;
10405 	nvlist_t **child;
10406 	uint_t c, children;
10407 	vdev_stat_t *vs;
10408 
10409 	assert(activity == ZPOOL_WAIT_INITIALIZE ||
10410 	    activity == ZPOOL_WAIT_TRIM);
10411 
10412 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
10413 	    (uint64_t **)&vs, &c) == 0);
10414 
10415 	if (activity == ZPOOL_WAIT_INITIALIZE &&
10416 	    vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE)
10417 		bytes_remaining = vs->vs_initialize_bytes_est -
10418 		    vs->vs_initialize_bytes_done;
10419 	else if (activity == ZPOOL_WAIT_TRIM &&
10420 	    vs->vs_trim_state == VDEV_TRIM_ACTIVE)
10421 		bytes_remaining = vs->vs_trim_bytes_est -
10422 		    vs->vs_trim_bytes_done;
10423 	else
10424 		bytes_remaining = 0;
10425 
10426 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
10427 	    &child, &children) != 0)
10428 		children = 0;
10429 
10430 	for (c = 0; c < children; c++)
10431 		bytes_remaining += vdev_activity_remaining(child[c], activity);
10432 
10433 	return (bytes_remaining);
10434 }
10435 
10436 /* Add up the total number of bytes left to rebuild across top-level vdevs */
10437 static uint64_t
10438 vdev_activity_top_remaining(nvlist_t *nv)
10439 {
10440 	uint64_t bytes_remaining = 0;
10441 	nvlist_t **child;
10442 	uint_t children;
10443 	int error;
10444 
10445 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
10446 	    &child, &children) != 0)
10447 		children = 0;
10448 
10449 	for (uint_t c = 0; c < children; c++) {
10450 		vdev_rebuild_stat_t *vrs;
10451 		uint_t i;
10452 
10453 		error = nvlist_lookup_uint64_array(child[c],
10454 		    ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i);
10455 		if (error == 0) {
10456 			if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
10457 				bytes_remaining += (vrs->vrs_bytes_est -
10458 				    vrs->vrs_bytes_rebuilt);
10459 			}
10460 		}
10461 	}
10462 
10463 	return (bytes_remaining);
10464 }
10465 
10466 /* Whether any vdevs are 'spare' or 'replacing' vdevs */
10467 static boolean_t
10468 vdev_any_spare_replacing(nvlist_t *nv)
10469 {
10470 	nvlist_t **child;
10471 	uint_t c, children;
10472 	char *vdev_type;
10473 
10474 	(void) nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &vdev_type);
10475 
10476 	if (strcmp(vdev_type, VDEV_TYPE_REPLACING) == 0 ||
10477 	    strcmp(vdev_type, VDEV_TYPE_SPARE) == 0 ||
10478 	    strcmp(vdev_type, VDEV_TYPE_DRAID_SPARE) == 0) {
10479 		return (B_TRUE);
10480 	}
10481 
10482 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
10483 	    &child, &children) != 0)
10484 		children = 0;
10485 
10486 	for (c = 0; c < children; c++) {
10487 		if (vdev_any_spare_replacing(child[c]))
10488 			return (B_TRUE);
10489 	}
10490 
10491 	return (B_FALSE);
10492 }
10493 
10494 typedef struct wait_data {
10495 	char *wd_poolname;
10496 	boolean_t wd_scripted;
10497 	boolean_t wd_exact;
10498 	boolean_t wd_headers_once;
10499 	boolean_t wd_should_exit;
10500 	/* Which activities to wait for */
10501 	boolean_t wd_enabled[ZPOOL_WAIT_NUM_ACTIVITIES];
10502 	float wd_interval;
10503 	pthread_cond_t wd_cv;
10504 	pthread_mutex_t wd_mutex;
10505 } wait_data_t;
10506 
10507 /*
10508  * Print to stdout a single line, containing one column for each activity that
10509  * we are waiting for specifying how many bytes of work are left for that
10510  * activity.
10511  */
10512 static void
10513 print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
10514 {
10515 	nvlist_t *config, *nvroot;
10516 	uint_t c;
10517 	int i;
10518 	pool_checkpoint_stat_t *pcs = NULL;
10519 	pool_scan_stat_t *pss = NULL;
10520 	pool_removal_stat_t *prs = NULL;
10521 	char *headers[] = {"DISCARD", "FREE", "INITIALIZE", "REPLACE",
10522 	    "REMOVE", "RESILVER", "SCRUB", "TRIM"};
10523 	int col_widths[ZPOOL_WAIT_NUM_ACTIVITIES];
10524 
10525 	/* Calculate the width of each column */
10526 	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10527 		/*
10528 		 * Make sure we have enough space in the col for pretty-printed
10529 		 * numbers and for the column header, and then leave a couple
10530 		 * spaces between cols for readability.
10531 		 */
10532 		col_widths[i] = MAX(strlen(headers[i]), 6) + 2;
10533 	}
10534 
10535 	/* Print header if appropriate */
10536 	int term_height = terminal_height();
10537 	boolean_t reprint_header = (!wd->wd_headers_once && term_height > 0 &&
10538 	    row % (term_height-1) == 0);
10539 	if (!wd->wd_scripted && (row == 0 || reprint_header)) {
10540 		for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10541 			if (wd->wd_enabled[i])
10542 				(void) printf("%*s", col_widths[i], headers[i]);
10543 		}
10544 		(void) printf("\n");
10545 	}
10546 
10547 	/* Bytes of work remaining in each activity */
10548 	int64_t bytes_rem[ZPOOL_WAIT_NUM_ACTIVITIES] = {0};
10549 
10550 	bytes_rem[ZPOOL_WAIT_FREE] =
10551 	    zpool_get_prop_int(zhp, ZPOOL_PROP_FREEING, NULL);
10552 
10553 	config = zpool_get_config(zhp, NULL);
10554 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
10555 
10556 	(void) nvlist_lookup_uint64_array(nvroot,
10557 	    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
10558 	if (pcs != NULL && pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
10559 		bytes_rem[ZPOOL_WAIT_CKPT_DISCARD] = pcs->pcs_space;
10560 
10561 	(void) nvlist_lookup_uint64_array(nvroot,
10562 	    ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
10563 	if (prs != NULL && prs->prs_state == DSS_SCANNING)
10564 		bytes_rem[ZPOOL_WAIT_REMOVE] = prs->prs_to_copy -
10565 		    prs->prs_copied;
10566 
10567 	(void) nvlist_lookup_uint64_array(nvroot,
10568 	    ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&pss, &c);
10569 	if (pss != NULL && pss->pss_state == DSS_SCANNING &&
10570 	    pss->pss_pass_scrub_pause == 0) {
10571 		int64_t rem = pss->pss_to_examine - pss->pss_issued;
10572 		if (pss->pss_func == POOL_SCAN_SCRUB)
10573 			bytes_rem[ZPOOL_WAIT_SCRUB] = rem;
10574 		else
10575 			bytes_rem[ZPOOL_WAIT_RESILVER] = rem;
10576 	} else if (check_rebuilding(nvroot, NULL)) {
10577 		bytes_rem[ZPOOL_WAIT_RESILVER] =
10578 		    vdev_activity_top_remaining(nvroot);
10579 	}
10580 
10581 	bytes_rem[ZPOOL_WAIT_INITIALIZE] =
10582 	    vdev_activity_remaining(nvroot, ZPOOL_WAIT_INITIALIZE);
10583 	bytes_rem[ZPOOL_WAIT_TRIM] =
10584 	    vdev_activity_remaining(nvroot, ZPOOL_WAIT_TRIM);
10585 
10586 	/*
10587 	 * A replace finishes after resilvering finishes, so the amount of work
10588 	 * left for a replace is the same as for resilvering.
10589 	 *
10590 	 * It isn't quite correct to say that if we have any 'spare' or
10591 	 * 'replacing' vdevs and a resilver is happening, then a replace is in
10592 	 * progress, like we do here. When a hot spare is used, the faulted vdev
10593 	 * is not removed after the hot spare is resilvered, so parent 'spare'
10594 	 * vdev is not removed either. So we could have a 'spare' vdev, but be
10595 	 * resilvering for a different reason. However, we use it as a heuristic
10596 	 * because we don't have access to the DTLs, which could tell us whether
10597 	 * or not we have really finished resilvering a hot spare.
10598 	 */
10599 	if (vdev_any_spare_replacing(nvroot))
10600 		bytes_rem[ZPOOL_WAIT_REPLACE] =  bytes_rem[ZPOOL_WAIT_RESILVER];
10601 
10602 	if (timestamp_fmt != NODATE)
10603 		print_timestamp(timestamp_fmt);
10604 
10605 	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10606 		char buf[64];
10607 		if (!wd->wd_enabled[i])
10608 			continue;
10609 
10610 		if (wd->wd_exact)
10611 			(void) snprintf(buf, sizeof (buf), "%" PRIi64,
10612 			    bytes_rem[i]);
10613 		else
10614 			zfs_nicenum(bytes_rem[i], buf, sizeof (buf));
10615 
10616 		if (wd->wd_scripted)
10617 			(void) printf(i == 0 ? "%s" : "\t%s", buf);
10618 		else
10619 			(void) printf(" %*s", col_widths[i] - 1, buf);
10620 	}
10621 	(void) printf("\n");
10622 	(void) fflush(stdout);
10623 }
10624 
10625 static void *
10626 wait_status_thread(void *arg)
10627 {
10628 	wait_data_t *wd = (wait_data_t *)arg;
10629 	zpool_handle_t *zhp;
10630 
10631 	if ((zhp = zpool_open(g_zfs, wd->wd_poolname)) == NULL)
10632 		return (void *)(1);
10633 
10634 	for (int row = 0; ; row++) {
10635 		boolean_t missing;
10636 		struct timespec timeout;
10637 		int ret = 0;
10638 		(void) clock_gettime(CLOCK_REALTIME, &timeout);
10639 
10640 		if (zpool_refresh_stats(zhp, &missing) != 0 || missing ||
10641 		    zpool_props_refresh(zhp) != 0) {
10642 			zpool_close(zhp);
10643 			return (void *)(uintptr_t)(missing ? 0 : 1);
10644 		}
10645 
10646 		print_wait_status_row(wd, zhp, row);
10647 
10648 		timeout.tv_sec += floor(wd->wd_interval);
10649 		long nanos = timeout.tv_nsec +
10650 		    (wd->wd_interval - floor(wd->wd_interval)) * NANOSEC;
10651 		if (nanos >= NANOSEC) {
10652 			timeout.tv_sec++;
10653 			timeout.tv_nsec = nanos - NANOSEC;
10654 		} else {
10655 			timeout.tv_nsec = nanos;
10656 		}
10657 		pthread_mutex_lock(&wd->wd_mutex);
10658 		if (!wd->wd_should_exit)
10659 			ret = pthread_cond_timedwait(&wd->wd_cv, &wd->wd_mutex,
10660 			    &timeout);
10661 		pthread_mutex_unlock(&wd->wd_mutex);
10662 		if (ret == 0) {
10663 			break; /* signaled by main thread */
10664 		} else if (ret != ETIMEDOUT) {
10665 			(void) fprintf(stderr, gettext("pthread_cond_timedwait "
10666 			    "failed: %s\n"), strerror(ret));
10667 			zpool_close(zhp);
10668 			return (void *)(uintptr_t)(1);
10669 		}
10670 	}
10671 
10672 	zpool_close(zhp);
10673 	return (void *)(0);
10674 }
10675 
10676 int
10677 zpool_do_wait(int argc, char **argv)
10678 {
10679 	boolean_t verbose = B_FALSE;
10680 	int c;
10681 	char *value;
10682 	int i;
10683 	unsigned long count;
10684 	pthread_t status_thr;
10685 	int error = 0;
10686 	zpool_handle_t *zhp;
10687 
10688 	wait_data_t wd;
10689 	wd.wd_scripted = B_FALSE;
10690 	wd.wd_exact = B_FALSE;
10691 	wd.wd_headers_once = B_FALSE;
10692 	wd.wd_should_exit = B_FALSE;
10693 
10694 	pthread_mutex_init(&wd.wd_mutex, NULL);
10695 	pthread_cond_init(&wd.wd_cv, NULL);
10696 
10697 	/* By default, wait for all types of activity. */
10698 	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++)
10699 		wd.wd_enabled[i] = B_TRUE;
10700 
10701 	while ((c = getopt(argc, argv, "HpT:t:")) != -1) {
10702 		switch (c) {
10703 		case 'H':
10704 			wd.wd_scripted = B_TRUE;
10705 			break;
10706 		case 'n':
10707 			wd.wd_headers_once = B_TRUE;
10708 			break;
10709 		case 'p':
10710 			wd.wd_exact = B_TRUE;
10711 			break;
10712 		case 'T':
10713 			get_timestamp_arg(*optarg);
10714 			break;
10715 		case 't':
10716 		{
10717 			static char *col_subopts[] = { "discard", "free",
10718 			    "initialize", "replace", "remove", "resilver",
10719 			    "scrub", "trim", NULL };
10720 
10721 			/* Reset activities array */
10722 			bzero(&wd.wd_enabled, sizeof (wd.wd_enabled));
10723 			while (*optarg != '\0') {
10724 				int activity = getsubopt(&optarg, col_subopts,
10725 				    &value);
10726 
10727 				if (activity < 0) {
10728 					(void) fprintf(stderr,
10729 					    gettext("invalid activity '%s'\n"),
10730 					    value);
10731 					usage(B_FALSE);
10732 				}
10733 
10734 				wd.wd_enabled[activity] = B_TRUE;
10735 			}
10736 			break;
10737 		}
10738 		case '?':
10739 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
10740 			    optopt);
10741 			usage(B_FALSE);
10742 		}
10743 	}
10744 
10745 	argc -= optind;
10746 	argv += optind;
10747 
10748 	get_interval_count(&argc, argv, &wd.wd_interval, &count);
10749 	if (count != 0) {
10750 		/* This subcmd only accepts an interval, not a count */
10751 		(void) fprintf(stderr, gettext("too many arguments\n"));
10752 		usage(B_FALSE);
10753 	}
10754 
10755 	if (wd.wd_interval != 0)
10756 		verbose = B_TRUE;
10757 
10758 	if (argc < 1) {
10759 		(void) fprintf(stderr, gettext("missing 'pool' argument\n"));
10760 		usage(B_FALSE);
10761 	}
10762 	if (argc > 1) {
10763 		(void) fprintf(stderr, gettext("too many arguments\n"));
10764 		usage(B_FALSE);
10765 	}
10766 
10767 	wd.wd_poolname = argv[0];
10768 
10769 	if ((zhp = zpool_open(g_zfs, wd.wd_poolname)) == NULL)
10770 		return (1);
10771 
10772 	if (verbose) {
10773 		/*
10774 		 * We use a separate thread for printing status updates because
10775 		 * the main thread will call lzc_wait(), which blocks as long
10776 		 * as an activity is in progress, which can be a long time.
10777 		 */
10778 		if (pthread_create(&status_thr, NULL, wait_status_thread, &wd)
10779 		    != 0) {
10780 			(void) fprintf(stderr, gettext("failed to create status"
10781 			    "thread: %s\n"), strerror(errno));
10782 			zpool_close(zhp);
10783 			return (1);
10784 		}
10785 	}
10786 
10787 	/*
10788 	 * Loop over all activities that we are supposed to wait for until none
10789 	 * of them are in progress. Note that this means we can end up waiting
10790 	 * for more activities to complete than just those that were in progress
10791 	 * when we began waiting; if an activity we are interested in begins
10792 	 * while we are waiting for another activity, we will wait for both to
10793 	 * complete before exiting.
10794 	 */
10795 	for (;;) {
10796 		boolean_t missing = B_FALSE;
10797 		boolean_t any_waited = B_FALSE;
10798 
10799 		for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10800 			boolean_t waited;
10801 
10802 			if (!wd.wd_enabled[i])
10803 				continue;
10804 
10805 			error = zpool_wait_status(zhp, i, &missing, &waited);
10806 			if (error != 0 || missing)
10807 				break;
10808 
10809 			any_waited = (any_waited || waited);
10810 		}
10811 
10812 		if (error != 0 || missing || !any_waited)
10813 			break;
10814 	}
10815 
10816 	zpool_close(zhp);
10817 
10818 	if (verbose) {
10819 		uintptr_t status;
10820 		pthread_mutex_lock(&wd.wd_mutex);
10821 		wd.wd_should_exit = B_TRUE;
10822 		pthread_cond_signal(&wd.wd_cv);
10823 		pthread_mutex_unlock(&wd.wd_mutex);
10824 		(void) pthread_join(status_thr, (void *)&status);
10825 		if (status != 0)
10826 			error = status;
10827 	}
10828 
10829 	pthread_mutex_destroy(&wd.wd_mutex);
10830 	pthread_cond_destroy(&wd.wd_cv);
10831 	return (error);
10832 }
10833 
10834 static int
10835 find_command_idx(char *command, int *idx)
10836 {
10837 	int i;
10838 
10839 	for (i = 0; i < NCOMMAND; i++) {
10840 		if (command_table[i].name == NULL)
10841 			continue;
10842 
10843 		if (strcmp(command, command_table[i].name) == 0) {
10844 			*idx = i;
10845 			return (0);
10846 		}
10847 	}
10848 	return (1);
10849 }
10850 
10851 /*
10852  * Display version message
10853  */
10854 static int
10855 zpool_do_version(int argc, char **argv)
10856 {
10857 	if (zfs_version_print() == -1)
10858 		return (1);
10859 
10860 	return (0);
10861 }
10862 
10863 /*
10864  * Do zpool_load_compat() and print error message on failure
10865  */
10866 static zpool_compat_status_t
10867 zpool_do_load_compat(const char *compat, boolean_t *list)
10868 {
10869 	char report[1024];
10870 
10871 	zpool_compat_status_t ret;
10872 
10873 	ret = zpool_load_compat(compat, list, report, 1024);
10874 	switch (ret) {
10875 
10876 	case ZPOOL_COMPATIBILITY_OK:
10877 		break;
10878 
10879 	case ZPOOL_COMPATIBILITY_NOFILES:
10880 	case ZPOOL_COMPATIBILITY_BADFILE:
10881 	case ZPOOL_COMPATIBILITY_BADTOKEN:
10882 		(void) fprintf(stderr, "Error: %s\n", report);
10883 		break;
10884 
10885 	case ZPOOL_COMPATIBILITY_WARNTOKEN:
10886 		(void) fprintf(stderr, "Warning: %s\n", report);
10887 		ret = ZPOOL_COMPATIBILITY_OK;
10888 		break;
10889 	}
10890 	return (ret);
10891 }
10892 
10893 int
10894 main(int argc, char **argv)
10895 {
10896 	int ret = 0;
10897 	int i = 0;
10898 	char *cmdname;
10899 	char **newargv;
10900 
10901 	(void) setlocale(LC_ALL, "");
10902 	(void) setlocale(LC_NUMERIC, "C");
10903 	(void) textdomain(TEXT_DOMAIN);
10904 	srand(time(NULL));
10905 
10906 	opterr = 0;
10907 
10908 	/*
10909 	 * Make sure the user has specified some command.
10910 	 */
10911 	if (argc < 2) {
10912 		(void) fprintf(stderr, gettext("missing command\n"));
10913 		usage(B_FALSE);
10914 	}
10915 
10916 	cmdname = argv[1];
10917 
10918 	/*
10919 	 * Special case '-?'
10920 	 */
10921 	if ((strcmp(cmdname, "-?") == 0) || strcmp(cmdname, "--help") == 0)
10922 		usage(B_TRUE);
10923 
10924 	/*
10925 	 * Special case '-V|--version'
10926 	 */
10927 	if ((strcmp(cmdname, "-V") == 0) || (strcmp(cmdname, "--version") == 0))
10928 		return (zpool_do_version(argc, argv));
10929 
10930 	if ((g_zfs = libzfs_init()) == NULL) {
10931 		(void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
10932 		return (1);
10933 	}
10934 
10935 	libzfs_print_on_error(g_zfs, B_TRUE);
10936 
10937 	zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
10938 
10939 	/*
10940 	 * Many commands modify input strings for string parsing reasons.
10941 	 * We create a copy to protect the original argv.
10942 	 */
10943 	newargv = malloc((argc + 1) * sizeof (newargv[0]));
10944 	for (i = 0; i < argc; i++)
10945 		newargv[i] = strdup(argv[i]);
10946 	newargv[argc] = NULL;
10947 
10948 	/*
10949 	 * Run the appropriate command.
10950 	 */
10951 	if (find_command_idx(cmdname, &i) == 0) {
10952 		current_command = &command_table[i];
10953 		ret = command_table[i].func(argc - 1, newargv + 1);
10954 	} else if (strchr(cmdname, '=')) {
10955 		verify(find_command_idx("set", &i) == 0);
10956 		current_command = &command_table[i];
10957 		ret = command_table[i].func(argc, newargv);
10958 	} else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
10959 		/*
10960 		 * 'freeze' is a vile debugging abomination, so we treat
10961 		 * it as such.
10962 		 */
10963 		zfs_cmd_t zc = {"\0"};
10964 
10965 		(void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
10966 		ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
10967 		if (ret != 0) {
10968 			(void) fprintf(stderr,
10969 			gettext("failed to freeze pool: %d\n"), errno);
10970 			ret = 1;
10971 		}
10972 
10973 		log_history = 0;
10974 	} else {
10975 		(void) fprintf(stderr, gettext("unrecognized "
10976 		    "command '%s'\n"), cmdname);
10977 		usage(B_FALSE);
10978 		ret = 1;
10979 	}
10980 
10981 	for (i = 0; i < argc; i++)
10982 		free(newargv[i]);
10983 	free(newargv);
10984 
10985 	if (ret == 0 && log_history)
10986 		(void) zpool_log_history(g_zfs, history_str);
10987 
10988 	libzfs_fini(g_zfs);
10989 
10990 	/*
10991 	 * The 'ZFS_ABORT' environment variable causes us to dump core on exit
10992 	 * for the purposes of running ::findleaks.
10993 	 */
10994 	if (getenv("ZFS_ABORT") != NULL) {
10995 		(void) printf("dumping core by request\n");
10996 		abort();
10997 	}
10998 
10999 	return (ret);
11000 }
11001