xref: /illumos-gate/usr/src/cmd/zpool/zpool_main.c (revision a92282e44f968185a6bba094d1e5fece2da819cf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
25  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
26  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
27  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
28  * Copyright 2016 Nexenta Systems, Inc.
29  * Copyright (c) 2017 Datto Inc.
30  * Copyright (c) 2017, Intel Corporation.
31  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
32  * Copyright 2020 Joyent, Inc.
33  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
34  * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
35  */
36 
37 #include <assert.h>
38 #include <ctype.h>
39 #include <dirent.h>
40 #include <errno.h>
41 #include <fcntl.h>
42 #include <getopt.h>
43 #include <libgen.h>
44 #include <libintl.h>
45 #include <libuutil.h>
46 #include <locale.h>
47 #include <stdio.h>
48 #include <stdlib.h>
49 #include <string.h>
50 #include <strings.h>
51 #include <unistd.h>
52 #include <priv.h>
53 #include <pwd.h>
54 #include <zone.h>
55 #include <zfs_prop.h>
56 #include <sys/fs/zfs.h>
57 #include <sys/stat.h>
58 #include <sys/debug.h>
59 #include <math.h>
60 #include <sys/sysmacros.h>
61 #include <sys/termios.h>
62 
63 #include <libzfs.h>
64 #include <libzutil.h>
65 
66 #include "zpool_util.h"
67 #include "zfs_comutil.h"
68 #include "zfeature_common.h"
69 
70 #include "statcommon.h"
71 
72 static int zpool_do_create(int, char **);
73 static int zpool_do_destroy(int, char **);
74 
75 static int zpool_do_add(int, char **);
76 static int zpool_do_remove(int, char **);
77 static int zpool_do_labelclear(int, char **);
78 
79 static int zpool_do_checkpoint(int, char **);
80 
81 static int zpool_do_list(int, char **);
82 static int zpool_do_iostat(int, char **);
83 static int zpool_do_status(int, char **);
84 
85 static int zpool_do_online(int, char **);
86 static int zpool_do_offline(int, char **);
87 static int zpool_do_clear(int, char **);
88 static int zpool_do_reopen(int, char **);
89 
90 static int zpool_do_reguid(int, char **);
91 
92 static int zpool_do_attach(int, char **);
93 static int zpool_do_detach(int, char **);
94 static int zpool_do_replace(int, char **);
95 static int zpool_do_split(int, char **);
96 
97 static int zpool_do_initialize(int, char **);
98 static int zpool_do_scrub(int, char **);
99 static int zpool_do_resilver(int, char **);
100 static int zpool_do_trim(int, char **);
101 
102 static int zpool_do_import(int, char **);
103 static int zpool_do_export(int, char **);
104 
105 static int zpool_do_upgrade(int, char **);
106 
107 static int zpool_do_history(int, char **);
108 
109 static int zpool_do_get(int, char **);
110 static int zpool_do_set(int, char **);
111 
112 static int zpool_do_sync(int, char **);
113 
114 /*
115  * These libumem hooks provide a reasonable set of defaults for the allocator's
116  * debugging facilities.
117  */
118 
119 #ifdef DEBUG
120 const char *
121 _umem_debug_init(void)
122 {
123 	return ("default,verbose"); /* $UMEM_DEBUG setting */
124 }
125 
126 const char *
127 _umem_logging_init(void)
128 {
129 	return ("fail,contents"); /* $UMEM_LOGGING setting */
130 }
131 #endif
132 
133 typedef enum {
134 	HELP_ADD,
135 	HELP_ATTACH,
136 	HELP_CLEAR,
137 	HELP_CREATE,
138 	HELP_CHECKPOINT,
139 	HELP_DESTROY,
140 	HELP_DETACH,
141 	HELP_EXPORT,
142 	HELP_HISTORY,
143 	HELP_IMPORT,
144 	HELP_IOSTAT,
145 	HELP_LABELCLEAR,
146 	HELP_LIST,
147 	HELP_OFFLINE,
148 	HELP_ONLINE,
149 	HELP_REPLACE,
150 	HELP_REMOVE,
151 	HELP_INITIALIZE,
152 	HELP_SCRUB,
153 	HELP_RESILVER,
154 	HELP_TRIM,
155 	HELP_STATUS,
156 	HELP_UPGRADE,
157 	HELP_GET,
158 	HELP_SET,
159 	HELP_SPLIT,
160 	HELP_SYNC,
161 	HELP_REGUID,
162 	HELP_REOPEN
163 } zpool_help_t;
164 
165 
166 /*
167  * Flags for stats to display with "zpool iostats"
168  */
169 enum iostat_type {
170 	IOS_DEFAULT = 0,
171 	IOS_LATENCY = 1,
172 	IOS_QUEUES = 2,
173 	IOS_L_HISTO = 3,
174 	IOS_RQ_HISTO = 4,
175 	IOS_COUNT,	/* always last element */
176 };
177 
178 /* iostat_type entries as bitmasks */
179 #define	IOS_DEFAULT_M	(1ULL << IOS_DEFAULT)
180 #define	IOS_LATENCY_M	(1ULL << IOS_LATENCY)
181 #define	IOS_QUEUES_M	(1ULL << IOS_QUEUES)
182 #define	IOS_L_HISTO_M	(1ULL << IOS_L_HISTO)
183 #define	IOS_RQ_HISTO_M	(1ULL << IOS_RQ_HISTO)
184 
185 /* Mask of all the histo bits */
186 #define	IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M)
187 
188 /*
189  * Lookup table for iostat flags to nvlist names.  Basically a list
190  * of all the nvlists a flag requires.  Also specifies the order in
191  * which data gets printed in zpool iostat.
192  */
193 static const char *vsx_type_to_nvlist[IOS_COUNT][13] = {
194 	[IOS_L_HISTO] = {
195 	    ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
196 	    ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
197 	    ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
198 	    ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
199 	    ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
200 	    ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
201 	    ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
202 	    ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
203 	    ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
204 	    ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
205 	    NULL},
206 	[IOS_LATENCY] = {
207 	    ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
208 	    ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
209 	    ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
210 	    ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
211 	    ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
212 	    NULL},
213 	[IOS_QUEUES] = {
214 	    ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
215 	    ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
216 	    ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
217 	    ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
218 	    ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
219 	    ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
220 	    NULL},
221 	[IOS_RQ_HISTO] = {
222 	    ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
223 	    ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
224 	    ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
225 	    ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
226 	    ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
227 	    ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
228 	    ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
229 	    ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
230 	    ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
231 	    ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
232 	    ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
233 	    ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
234 	    NULL},
235 };
236 
237 
238 /*
239  * Given a cb->cb_flags with a histogram bit set, return the iostat_type.
240  * Right now, only one histo bit is ever set at one time, so we can
241  * just do a highbit64(a)
242  */
243 #define	IOS_HISTO_IDX(a)	(highbit64(a & IOS_ANYHISTO_M) - 1)
244 
245 typedef struct zpool_command {
246 	const char	*name;
247 	int		(*func)(int, char **);
248 	zpool_help_t	usage;
249 } zpool_command_t;
250 
251 /*
252  * Master command table.  Each ZFS command has a name, associated function, and
253  * usage message.  The usage messages need to be internationalized, so we have
254  * to have a function to return the usage message based on a command index.
255  *
256  * These commands are organized according to how they are displayed in the usage
257  * message.  An empty command (one with a NULL name) indicates an empty line in
258  * the generic usage message.
259  */
260 static zpool_command_t command_table[] = {
261 	{ "create",	zpool_do_create,	HELP_CREATE		},
262 	{ "destroy",	zpool_do_destroy,	HELP_DESTROY		},
263 	{ NULL },
264 	{ "add",	zpool_do_add,		HELP_ADD		},
265 	{ "remove",	zpool_do_remove,	HELP_REMOVE		},
266 	{ NULL },
267 	{ "labelclear",	zpool_do_labelclear,	HELP_LABELCLEAR		},
268 	{ NULL },
269 	{ "checkpoint",	zpool_do_checkpoint,	HELP_CHECKPOINT		},
270 	{ NULL },
271 	{ "list",	zpool_do_list,		HELP_LIST		},
272 	{ "iostat",	zpool_do_iostat,	HELP_IOSTAT		},
273 	{ "status",	zpool_do_status,	HELP_STATUS		},
274 	{ NULL },
275 	{ "online",	zpool_do_online,	HELP_ONLINE		},
276 	{ "offline",	zpool_do_offline,	HELP_OFFLINE		},
277 	{ "clear",	zpool_do_clear,		HELP_CLEAR		},
278 	{ "reopen",	zpool_do_reopen,	HELP_REOPEN		},
279 	{ NULL },
280 	{ "attach",	zpool_do_attach,	HELP_ATTACH		},
281 	{ "detach",	zpool_do_detach,	HELP_DETACH		},
282 	{ "replace",	zpool_do_replace,	HELP_REPLACE		},
283 	{ "split",	zpool_do_split,		HELP_SPLIT		},
284 	{ NULL },
285 	{ "initialize",	zpool_do_initialize,	HELP_INITIALIZE		},
286 	{ "resilver",	zpool_do_resilver,	HELP_RESILVER		},
287 	{ "scrub",	zpool_do_scrub,		HELP_SCRUB		},
288 	{ "trim",	zpool_do_trim,		HELP_TRIM		},
289 	{ NULL },
290 	{ "import",	zpool_do_import,	HELP_IMPORT		},
291 	{ "export",	zpool_do_export,	HELP_EXPORT		},
292 	{ "upgrade",	zpool_do_upgrade,	HELP_UPGRADE		},
293 	{ "reguid",	zpool_do_reguid,	HELP_REGUID		},
294 	{ NULL },
295 	{ "history",	zpool_do_history,	HELP_HISTORY		},
296 	{ "get",	zpool_do_get,		HELP_GET		},
297 	{ "set",	zpool_do_set,		HELP_SET		},
298 	{ "sync",	zpool_do_sync,		HELP_SYNC		},
299 };
300 
301 #define	NCOMMAND	(ARRAY_SIZE(command_table))
302 
303 #define	VDEV_ALLOC_CLASS_LOGS	"logs"
304 
305 static zpool_command_t *current_command;
306 static char history_str[HIS_MAX_RECORD_LEN];
307 static boolean_t log_history = B_TRUE;
308 static uint_t timestamp_fmt = NODATE;
309 
310 static const char *
311 get_usage(zpool_help_t idx)
312 {
313 	switch (idx) {
314 	case HELP_ADD:
315 		return (gettext("\tadd [-fgLnP] [-o property=value] "
316 		    "<pool> <vdev> ...\n"));
317 	case HELP_ATTACH:
318 		return (gettext("\tattach [-f] [-o property=value] "
319 		    "<pool> <device> <new-device>\n"));
320 	case HELP_CLEAR:
321 		return (gettext("\tclear [-nF] <pool> [device]\n"));
322 	case HELP_CREATE:
323 		return (gettext("\tcreate [-fnd] [-B] "
324 		    "[-o property=value] ... \n"
325 		    "\t    [-O file-system-property=value] ...\n"
326 		    "\t    [-m mountpoint] [-R root] [-t tempname] "
327 		    "<pool> <vdev> ...\n"));
328 	case HELP_CHECKPOINT:
329 		return (gettext("\tcheckpoint [--discard] <pool> ...\n"));
330 	case HELP_DESTROY:
331 		return (gettext("\tdestroy [-f] <pool>\n"));
332 	case HELP_DETACH:
333 		return (gettext("\tdetach <pool> <device>\n"));
334 	case HELP_EXPORT:
335 		return (gettext("\texport [-f] <pool> ...\n"));
336 	case HELP_HISTORY:
337 		return (gettext("\thistory [-il] [<pool>] ...\n"));
338 	case HELP_IMPORT:
339 		return (gettext("\timport [-d dir] [-D]\n"
340 		    "\timport [-d dir | -c cachefile] [-F [-n]] [-l] "
341 		    "<pool | id>\n"
342 		    "\timport [-o mntopts] [-o property=value] ... \n"
343 		    "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
344 		    "[-R root] [-F [-n]] -a\n"
345 		    "\timport [-o mntopts] [-o property=value] ... \n"
346 		    "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
347 		    "[-R root] [-F [-n]] [-t]\n"
348 		    "\t    [--rewind-to-checkpoint] <pool | id> [newpool]\n"));
349 	case HELP_IOSTAT:
350 		return (gettext("\tiostat "
351 		    "[[-lq]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
352 		    "\t    [[pool] ...]|[pool vdev ...]|[vdev ...]]"
353 		    " [[-n] interval [count]]\n"));
354 	case HELP_LABELCLEAR:
355 		return (gettext("\tlabelclear [-f] <vdev>\n"));
356 	case HELP_LIST:
357 		return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
358 		    "[-T d|u] [pool] ... [interval [count]]\n"));
359 	case HELP_OFFLINE:
360 		return (gettext("\toffline [-t] <pool> <device> ...\n"));
361 	case HELP_ONLINE:
362 		return (gettext("\tonline <pool> <device> ...\n"));
363 	case HELP_REPLACE:
364 		return (gettext("\treplace [-f] <pool> <device> "
365 		    "[new-device]\n"));
366 	case HELP_REMOVE:
367 		return (gettext("\tremove [-nps] <pool> <device> ...\n"));
368 	case HELP_REOPEN:
369 		return (gettext("\treopen <pool>\n"));
370 	case HELP_INITIALIZE:
371 		return (gettext("\tinitialize [-c | -s] <pool> "
372 		    "[<device> ...]\n"));
373 	case HELP_SCRUB:
374 		return (gettext("\tscrub [-s | -p] <pool> ...\n"));
375 	case HELP_RESILVER:
376 		return (gettext("\tresilver <pool> ...\n"));
377 	case HELP_TRIM:
378 		return (gettext("\ttrim [-d] [-r <rate>] [-c | -s] <pool> "
379 		    "[<device> ...]\n"));
380 	case HELP_STATUS:
381 		return (gettext("\tstatus "
382 		    "[-igLpPsvxD] [-T d|u] [pool] ... "
383 		    "[interval [count]]\n"));
384 	case HELP_UPGRADE:
385 		return (gettext("\tupgrade\n"
386 		    "\tupgrade -v\n"
387 		    "\tupgrade [-V version] <-a | pool ...>\n"));
388 	case HELP_GET:
389 		return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
390 		    "<\"all\" | property[,...]> <pool> ...\n"));
391 	case HELP_SET:
392 		return (gettext("\tset <property=value> <pool> \n"));
393 	case HELP_SPLIT:
394 		return (gettext("\tsplit [-gLlnP] [-R altroot] [-o mntopts]\n"
395 		    "\t    [-o property=value] <pool> <newpool> "
396 		    "[<device> ...]\n"));
397 	case HELP_REGUID:
398 		return (gettext("\treguid <pool>\n"));
399 	case HELP_SYNC:
400 		return (gettext("\tsync [pool] ...\n"));
401 	}
402 
403 	abort();
404 	/* NOTREACHED */
405 }
406 
407 static void
408 zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res)
409 {
410 	uint_t children = 0;
411 	nvlist_t **child;
412 	uint_t i;
413 
414 	(void) nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
415 	    &child, &children);
416 
417 	if (children == 0) {
418 		char *path = zpool_vdev_name(g_zfs, zhp, nvroot, 0);
419 
420 		if (strcmp(path, VDEV_TYPE_INDIRECT) != 0)
421 			fnvlist_add_boolean(res, path);
422 
423 		free(path);
424 		return;
425 	}
426 
427 	for (i = 0; i < children; i++) {
428 		zpool_collect_leaves(zhp, child[i], res);
429 	}
430 }
431 
432 /*
433  * Callback routine that will print out a pool property value.
434  */
435 static int
436 print_prop_cb(int prop, void *cb)
437 {
438 	FILE *fp = cb;
439 
440 	(void) fprintf(fp, "\t%-19s  ", zpool_prop_to_name(prop));
441 
442 	if (zpool_prop_readonly(prop))
443 		(void) fprintf(fp, "  NO   ");
444 	else
445 		(void) fprintf(fp, " YES   ");
446 
447 	if (zpool_prop_values(prop) == NULL)
448 		(void) fprintf(fp, "-\n");
449 	else
450 		(void) fprintf(fp, "%s\n", zpool_prop_values(prop));
451 
452 	return (ZPROP_CONT);
453 }
454 
455 /*
456  * Display usage message.  If we're inside a command, display only the usage for
457  * that command.  Otherwise, iterate over the entire command table and display
458  * a complete usage message.
459  */
460 void
461 usage(boolean_t requested)
462 {
463 	FILE *fp = requested ? stdout : stderr;
464 
465 	if (current_command == NULL) {
466 		int i;
467 
468 		(void) fprintf(fp, gettext("usage: zpool command args ...\n"));
469 		(void) fprintf(fp,
470 		    gettext("where 'command' is one of the following:\n\n"));
471 
472 		for (i = 0; i < NCOMMAND; i++) {
473 			if (command_table[i].name == NULL)
474 				(void) fprintf(fp, "\n");
475 			else
476 				(void) fprintf(fp, "%s",
477 				    get_usage(command_table[i].usage));
478 		}
479 	} else {
480 		(void) fprintf(fp, gettext("usage:\n"));
481 		(void) fprintf(fp, "%s", get_usage(current_command->usage));
482 	}
483 
484 	if (current_command != NULL &&
485 	    ((strcmp(current_command->name, "set") == 0) ||
486 	    (strcmp(current_command->name, "get") == 0) ||
487 	    (strcmp(current_command->name, "list") == 0))) {
488 
489 		(void) fprintf(fp,
490 		    gettext("\nthe following properties are supported:\n"));
491 
492 		(void) fprintf(fp, "\n\t%-19s  %s   %s\n\n",
493 		    "PROPERTY", "EDIT", "VALUES");
494 
495 		/* Iterate over all properties */
496 		(void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
497 		    ZFS_TYPE_POOL);
498 
499 		(void) fprintf(fp, "\t%-19s   ", "feature@...");
500 		(void) fprintf(fp, "YES   disabled | enabled | active\n");
501 
502 		(void) fprintf(fp, gettext("\nThe feature@ properties must be "
503 		    "appended with a feature name.\nSee zpool-features(5).\n"));
504 	}
505 
506 	/*
507 	 * See comments at end of main().
508 	 */
509 	if (getenv("ZFS_ABORT") != NULL) {
510 		(void) printf("dumping core by request\n");
511 		abort();
512 	}
513 
514 	exit(requested ? 0 : 2);
515 }
516 
517 /*
518  * print a pool vdev config for dry runs
519  */
520 static void
521 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
522     const char *match, int name_flags)
523 {
524 	nvlist_t **child;
525 	uint_t c, children;
526 	char *vname;
527 	boolean_t printed = B_FALSE;
528 
529 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
530 	    &child, &children) != 0) {
531 		if (name != NULL)
532 			(void) printf("\t%*s%s\n", indent, "", name);
533 		return;
534 	}
535 
536 	for (c = 0; c < children; c++) {
537 		uint64_t is_log = B_FALSE;
538 		char *class = "";
539 
540 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
541 		    &is_log);
542 		if (is_log)
543 			class = VDEV_ALLOC_BIAS_LOG;
544 		(void) nvlist_lookup_string(child[c],
545 		    ZPOOL_CONFIG_ALLOCATION_BIAS, &class);
546 		if (strcmp(match, class) != 0)
547 			continue;
548 
549 		if (!printed && name != NULL) {
550 			(void) printf("\t%*s%s\n", indent, "", name);
551 			printed = B_TRUE;
552 		}
553 		vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
554 		print_vdev_tree(zhp, vname, child[c], indent + 2, "",
555 		    name_flags);
556 		free(vname);
557 	}
558 }
559 
560 static boolean_t
561 prop_list_contains_feature(nvlist_t *proplist)
562 {
563 	nvpair_t *nvp;
564 	for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
565 	    nvp = nvlist_next_nvpair(proplist, nvp)) {
566 		if (zpool_prop_feature(nvpair_name(nvp)))
567 			return (B_TRUE);
568 	}
569 	return (B_FALSE);
570 }
571 
572 /*
573  * Add a property pair (name, string-value) into a property nvlist.
574  */
575 static int
576 add_prop_list(const char *propname, char *propval, nvlist_t **props,
577     boolean_t poolprop)
578 {
579 	zpool_prop_t prop = ZPOOL_PROP_INVAL;
580 	zfs_prop_t fprop;
581 	nvlist_t *proplist;
582 	const char *normnm;
583 	char *strval;
584 
585 	if (*props == NULL &&
586 	    nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
587 		(void) fprintf(stderr,
588 		    gettext("internal error: out of memory\n"));
589 		return (1);
590 	}
591 
592 	proplist = *props;
593 
594 	if (poolprop) {
595 		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
596 
597 		if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
598 		    !zpool_prop_feature(propname)) {
599 			(void) fprintf(stderr, gettext("property '%s' is "
600 			    "not a valid pool property\n"), propname);
601 			return (2);
602 		}
603 
604 		/*
605 		 * feature@ properties and version should not be specified
606 		 * at the same time.
607 		 */
608 		if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
609 		    nvlist_exists(proplist, vname)) ||
610 		    (prop == ZPOOL_PROP_VERSION &&
611 		    prop_list_contains_feature(proplist))) {
612 			(void) fprintf(stderr, gettext("'feature@' and "
613 			    "'version' properties cannot be specified "
614 			    "together\n"));
615 			return (2);
616 		}
617 
618 
619 		if (zpool_prop_feature(propname))
620 			normnm = propname;
621 		else
622 			normnm = zpool_prop_to_name(prop);
623 	} else {
624 		if ((fprop = zfs_name_to_prop(propname)) != ZPROP_INVAL) {
625 			normnm = zfs_prop_to_name(fprop);
626 		} else {
627 			normnm = propname;
628 		}
629 	}
630 
631 	if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
632 	    prop != ZPOOL_PROP_CACHEFILE) {
633 		(void) fprintf(stderr, gettext("property '%s' "
634 		    "specified multiple times\n"), propname);
635 		return (2);
636 	}
637 
638 	if (nvlist_add_string(proplist, normnm, propval) != 0) {
639 		(void) fprintf(stderr, gettext("internal "
640 		    "error: out of memory\n"));
641 		return (1);
642 	}
643 
644 	return (0);
645 }
646 
647 /*
648  * Set a default property pair (name, string-value) in a property nvlist
649  */
650 static int
651 add_prop_list_default(const char *propname, char *propval, nvlist_t **props,
652     boolean_t poolprop)
653 {
654 	char *pval;
655 
656 	if (nvlist_lookup_string(*props, propname, &pval) == 0)
657 		return (0);
658 
659 	return (add_prop_list(propname, propval, props, poolprop));
660 }
661 
662 /*
663  * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
664  *
665  *	-f	Force addition of devices, even if they appear in use
666  *	-g	Display guid for individual vdev name.
667  *	-L	Follow links when resolving vdev path name.
668  *	-n	Do not add the devices, but display the resulting layout if
669  *		they were to be added.
670  *	-P	Display full path for vdev name.
671  *	-o	Set property=value.
672  *
673  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
674  * handled by get_vdev_spec(), which constructs the nvlist needed to pass to
675  * libzfs.
676  */
677 int
678 zpool_do_add(int argc, char **argv)
679 {
680 	boolean_t force = B_FALSE;
681 	boolean_t dryrun = B_FALSE;
682 	int name_flags = 0;
683 	int c;
684 	nvlist_t *nvroot;
685 	char *poolname;
686 	zpool_boot_label_t boot_type;
687 	uint64_t boot_size;
688 	int ret;
689 	zpool_handle_t *zhp;
690 	nvlist_t *config;
691 	nvlist_t *props = NULL;
692 	char *propval;
693 
694 	/* check options */
695 	while ((c = getopt(argc, argv, "fgLnPo:")) != -1) {
696 		switch (c) {
697 		case 'f':
698 			force = B_TRUE;
699 			break;
700 		case 'g':
701 			name_flags |= VDEV_NAME_GUID;
702 			break;
703 		case 'L':
704 			name_flags |= VDEV_NAME_FOLLOW_LINKS;
705 			break;
706 		case 'n':
707 			dryrun = B_TRUE;
708 			break;
709 		case 'P':
710 			name_flags |= VDEV_NAME_PATH;
711 			break;
712 		case 'o':
713 			if ((propval = strchr(optarg, '=')) == NULL) {
714 				(void) fprintf(stderr, gettext("missing "
715 				    "'=' for -o option\n"));
716 				usage(B_FALSE);
717 			}
718 			*propval = '\0';
719 			propval++;
720 
721 			if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
722 			    (add_prop_list(optarg, propval, &props, B_TRUE)))
723 				usage(B_FALSE);
724 			break;
725 		case '?':
726 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
727 			    optopt);
728 			usage(B_FALSE);
729 		}
730 	}
731 
732 	argc -= optind;
733 	argv += optind;
734 
735 	/* get pool name and check number of arguments */
736 	if (argc < 1) {
737 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
738 		usage(B_FALSE);
739 	}
740 	if (argc < 2) {
741 		(void) fprintf(stderr, gettext("missing vdev specification\n"));
742 		usage(B_FALSE);
743 	}
744 
745 	poolname = argv[0];
746 
747 	argc--;
748 	argv++;
749 
750 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
751 		return (1);
752 
753 	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
754 		(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
755 		    poolname);
756 		zpool_close(zhp);
757 		return (1);
758 	}
759 
760 	if (zpool_is_bootable(zhp))
761 		boot_type = ZPOOL_COPY_BOOT_LABEL;
762 	else
763 		boot_type = ZPOOL_NO_BOOT_LABEL;
764 
765 	/* unless manually specified use "ashift" pool property (if set) */
766 	if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
767 		int intval;
768 		zprop_source_t src;
769 		char strval[ZPOOL_MAXPROPLEN];
770 
771 		intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
772 		if (src != ZPROP_SRC_DEFAULT) {
773 			(void) sprintf(strval, "%" PRId32, intval);
774 			verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
775 			    &props, B_TRUE) == 0);
776 		}
777 	}
778 
779 	/* pass off to get_vdev_spec for processing */
780 	boot_size = zpool_get_prop_int(zhp, ZPOOL_PROP_BOOTSIZE, NULL);
781 	nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
782 	    boot_type, boot_size, argc, argv);
783 	if (nvroot == NULL) {
784 		zpool_close(zhp);
785 		return (1);
786 	}
787 
788 	if (dryrun) {
789 		nvlist_t *poolnvroot;
790 
791 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
792 		    &poolnvroot) == 0);
793 
794 		(void) printf(gettext("would update '%s' to the following "
795 		    "configuration:\n"), zpool_get_name(zhp));
796 
797 		/* print original main pool and new tree */
798 		print_vdev_tree(zhp, poolname, poolnvroot, 0, "",
799 		    name_flags | VDEV_NAME_TYPE_ID);
800 		print_vdev_tree(zhp, NULL, nvroot, 0, "", name_flags);
801 
802 		/* print other classes: 'dedup', 'special', and 'log' */
803 		print_vdev_tree(zhp, "dedup", poolnvroot, 0,
804 		    VDEV_ALLOC_BIAS_DEDUP, name_flags);
805 		print_vdev_tree(zhp, NULL, nvroot, 0, VDEV_ALLOC_BIAS_DEDUP,
806 		    name_flags);
807 
808 		print_vdev_tree(zhp, "special", poolnvroot, 0,
809 		    VDEV_ALLOC_BIAS_SPECIAL, name_flags);
810 		print_vdev_tree(zhp, NULL, nvroot, 0, VDEV_ALLOC_BIAS_SPECIAL,
811 		    name_flags);
812 
813 		print_vdev_tree(zhp, "logs", poolnvroot, 0, VDEV_ALLOC_BIAS_LOG,
814 		    name_flags);
815 		print_vdev_tree(zhp, NULL, nvroot, 0, VDEV_ALLOC_BIAS_LOG,
816 		    name_flags);
817 
818 		ret = 0;
819 	} else {
820 		ret = (zpool_add(zhp, nvroot) != 0);
821 	}
822 
823 	nvlist_free(props);
824 	nvlist_free(nvroot);
825 	zpool_close(zhp);
826 
827 	return (ret);
828 }
829 
830 /*
831  * zpool remove  <pool> <vdev> ...
832  *
833  * Removes the given vdev from the pool.
834  */
835 int
836 zpool_do_remove(int argc, char **argv)
837 {
838 	char *poolname;
839 	int i, ret = 0;
840 	zpool_handle_t *zhp;
841 	boolean_t stop = B_FALSE;
842 	boolean_t noop = B_FALSE;
843 	boolean_t parsable = B_FALSE;
844 	char c;
845 
846 	/* check options */
847 	while ((c = getopt(argc, argv, "nps")) != -1) {
848 		switch (c) {
849 		case 'n':
850 			noop = B_TRUE;
851 			break;
852 		case 'p':
853 			parsable = B_TRUE;
854 			break;
855 		case 's':
856 			stop = B_TRUE;
857 			break;
858 		case '?':
859 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
860 			    optopt);
861 			usage(B_FALSE);
862 		}
863 	}
864 
865 	argc -= optind;
866 	argv += optind;
867 
868 	/* get pool name and check number of arguments */
869 	if (argc < 1) {
870 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
871 		usage(B_FALSE);
872 	}
873 
874 	poolname = argv[0];
875 
876 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
877 		return (1);
878 
879 	if (stop && noop) {
880 		(void) fprintf(stderr, gettext("stop request ignored\n"));
881 		return (0);
882 	}
883 
884 	if (stop) {
885 		if (argc > 1) {
886 			(void) fprintf(stderr, gettext("too many arguments\n"));
887 			usage(B_FALSE);
888 		}
889 		if (zpool_vdev_remove_cancel(zhp) != 0)
890 			ret = 1;
891 	} else {
892 		if (argc < 2) {
893 			(void) fprintf(stderr, gettext("missing device\n"));
894 			usage(B_FALSE);
895 		}
896 
897 		for (i = 1; i < argc; i++) {
898 			if (noop) {
899 				uint64_t size;
900 
901 				if (zpool_vdev_indirect_size(zhp, argv[i],
902 				    &size) != 0) {
903 					ret = 1;
904 					break;
905 				}
906 				if (parsable) {
907 					(void) printf("%s %llu\n",
908 					    argv[i], size);
909 				} else {
910 					char valstr[32];
911 					zfs_nicenum(size, valstr,
912 					    sizeof (valstr));
913 					(void) printf("Memory that will be "
914 					    "used after removing %s: %s\n",
915 					    argv[i], valstr);
916 				}
917 			} else {
918 				if (zpool_vdev_remove(zhp, argv[i]) != 0)
919 					ret = 1;
920 			}
921 		}
922 	}
923 
924 	return (ret);
925 }
926 
927 /*
928  * zpool labelclear [-f] <vdev>
929  *
930  *	-f	Force clearing the label for the vdevs which are members of
931  *		the exported or foreign pools.
932  *
933  * Verifies that the vdev is not active and zeros out the label information
934  * on the device.
935  */
936 int
937 zpool_do_labelclear(int argc, char **argv)
938 {
939 	char vdev[MAXPATHLEN];
940 	char *name = NULL;
941 	struct stat st;
942 	int c, fd, ret = 0;
943 	nvlist_t *config;
944 	pool_state_t state;
945 	boolean_t inuse = B_FALSE;
946 	boolean_t force = B_FALSE;
947 
948 	/* check options */
949 	while ((c = getopt(argc, argv, "f")) != -1) {
950 		switch (c) {
951 		case 'f':
952 			force = B_TRUE;
953 			break;
954 		default:
955 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
956 			    optopt);
957 			usage(B_FALSE);
958 		}
959 	}
960 
961 	argc -= optind;
962 	argv += optind;
963 
964 	/* get vdev name */
965 	if (argc < 1) {
966 		(void) fprintf(stderr, gettext("missing vdev name\n"));
967 		usage(B_FALSE);
968 	}
969 	if (argc > 1) {
970 		(void) fprintf(stderr, gettext("too many arguments\n"));
971 		usage(B_FALSE);
972 	}
973 
974 	/*
975 	 * Check if we were given absolute path and use it as is.
976 	 * Otherwise if the provided vdev name doesn't point to a file,
977 	 * try prepending dsk path and appending s0.
978 	 */
979 	(void) strlcpy(vdev, argv[0], sizeof (vdev));
980 	if (vdev[0] != '/' && stat(vdev, &st) != 0) {
981 		char *s;
982 
983 		(void) snprintf(vdev, sizeof (vdev), "%s/%s",
984 		    ZFS_DISK_ROOT, argv[0]);
985 		if ((s = strrchr(argv[0], 's')) == NULL ||
986 		    !isdigit(*(s + 1)))
987 			(void) strlcat(vdev, "s0", sizeof (vdev));
988 		if (stat(vdev, &st) != 0) {
989 			(void) fprintf(stderr, gettext(
990 			    "failed to find device %s, try specifying absolute "
991 			    "path instead\n"), argv[0]);
992 			return (1);
993 		}
994 	}
995 
996 	if ((fd = open(vdev, O_RDWR)) < 0) {
997 		(void) fprintf(stderr, gettext("failed to open %s: %s\n"),
998 		    vdev, strerror(errno));
999 		return (1);
1000 	}
1001 
1002 	if (zpool_read_label(fd, &config, NULL) != 0) {
1003 		(void) fprintf(stderr,
1004 		    gettext("failed to read label from %s\n"), vdev);
1005 		return (1);
1006 	}
1007 	nvlist_free(config);
1008 
1009 	ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
1010 	if (ret != 0) {
1011 		(void) fprintf(stderr,
1012 		    gettext("failed to check state for %s\n"), vdev);
1013 		return (1);
1014 	}
1015 
1016 	if (!inuse)
1017 		goto wipe_label;
1018 
1019 	switch (state) {
1020 	default:
1021 	case POOL_STATE_ACTIVE:
1022 	case POOL_STATE_SPARE:
1023 	case POOL_STATE_L2CACHE:
1024 		(void) fprintf(stderr, gettext(
1025 		    "%s is a member (%s) of pool \"%s\"\n"),
1026 		    vdev, zpool_pool_state_to_name(state), name);
1027 		ret = 1;
1028 		goto errout;
1029 
1030 	case POOL_STATE_EXPORTED:
1031 		if (force)
1032 			break;
1033 		(void) fprintf(stderr, gettext(
1034 		    "use '-f' to override the following error:\n"
1035 		    "%s is a member of exported pool \"%s\"\n"),
1036 		    vdev, name);
1037 		ret = 1;
1038 		goto errout;
1039 
1040 	case POOL_STATE_POTENTIALLY_ACTIVE:
1041 		if (force)
1042 			break;
1043 		(void) fprintf(stderr, gettext(
1044 		    "use '-f' to override the following error:\n"
1045 		    "%s is a member of potentially active pool \"%s\"\n"),
1046 		    vdev, name);
1047 		ret = 1;
1048 		goto errout;
1049 
1050 	case POOL_STATE_DESTROYED:
1051 		/* inuse should never be set for a destroyed pool */
1052 		assert(0);
1053 		break;
1054 	}
1055 
1056 wipe_label:
1057 	ret = zpool_clear_label(fd);
1058 	if (ret != 0) {
1059 		(void) fprintf(stderr,
1060 		    gettext("failed to clear label for %s\n"), vdev);
1061 	}
1062 
1063 errout:
1064 	free(name);
1065 	(void) close(fd);
1066 
1067 	return (ret);
1068 }
1069 
1070 /*
1071  * zpool create [-fnd] [-B] [-o property=value] ...
1072  *		[-O file-system-property=value] ...
1073  *		[-R root] [-m mountpoint] [-t tempname] <pool> <dev> ...
1074  *
1075  *	-B	Create boot partition.
1076  *	-f	Force creation, even if devices appear in use
1077  *	-n	Do not create the pool, but display the resulting layout if it
1078  *		were to be created.
1079  *	-R	Create a pool under an alternate root
1080  *	-m	Set default mountpoint for the root dataset.  By default it's
1081  *		'/<pool>'
1082  *	-t	Use the temporary name until the pool is exported.
1083  *	-o	Set property=value.
1084  *	-o	Set feature@feature=enabled|disabled.
1085  *	-d	Don't automatically enable all supported pool features
1086  *		(individual features can be enabled with -o).
1087  *	-O	Set fsproperty=value in the pool's root file system
1088  *
1089  * Creates the named pool according to the given vdev specification.  The
1090  * bulk of the vdev processing is done in get_vdev_spec() in zpool_vdev.c.  Once
1091  * we get the nvlist back from get_vdev_spec(), we either print out the contents
1092  * (if '-n' was specified), or pass it to libzfs to do the creation.
1093  */
1094 
1095 #define	SYSTEM256	(256 * 1024 * 1024)
1096 int
1097 zpool_do_create(int argc, char **argv)
1098 {
1099 	boolean_t force = B_FALSE;
1100 	boolean_t dryrun = B_FALSE;
1101 	boolean_t enable_all_pool_feat = B_TRUE;
1102 	zpool_boot_label_t boot_type = ZPOOL_NO_BOOT_LABEL;
1103 	uint64_t boot_size = 0;
1104 	int c;
1105 	nvlist_t *nvroot = NULL;
1106 	char *poolname;
1107 	char *tname = NULL;
1108 	int ret = 1;
1109 	char *altroot = NULL;
1110 	char *mountpoint = NULL;
1111 	nvlist_t *fsprops = NULL;
1112 	nvlist_t *props = NULL;
1113 	char *propval;
1114 
1115 	/* check options */
1116 	while ((c = getopt(argc, argv, ":fndBR:m:o:O:t:")) != -1) {
1117 		switch (c) {
1118 		case 'f':
1119 			force = B_TRUE;
1120 			break;
1121 		case 'n':
1122 			dryrun = B_TRUE;
1123 			break;
1124 		case 'd':
1125 			enable_all_pool_feat = B_FALSE;
1126 			break;
1127 		case 'B':
1128 			/*
1129 			 * We should create the system partition.
1130 			 * Also make sure the size is set.
1131 			 */
1132 			boot_type = ZPOOL_CREATE_BOOT_LABEL;
1133 			if (boot_size == 0)
1134 				boot_size = SYSTEM256;
1135 			break;
1136 		case 'R':
1137 			altroot = optarg;
1138 			if (add_prop_list(zpool_prop_to_name(
1139 			    ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1140 				goto errout;
1141 			if (add_prop_list_default(zpool_prop_to_name(
1142 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1143 				goto errout;
1144 			break;
1145 		case 'm':
1146 			/* Equivalent to -O mountpoint=optarg */
1147 			mountpoint = optarg;
1148 			break;
1149 		case 'o':
1150 			if ((propval = strchr(optarg, '=')) == NULL) {
1151 				(void) fprintf(stderr, gettext("missing "
1152 				    "'=' for -o option\n"));
1153 				goto errout;
1154 			}
1155 			*propval = '\0';
1156 			propval++;
1157 
1158 			if (add_prop_list(optarg, propval, &props, B_TRUE))
1159 				goto errout;
1160 
1161 			/*
1162 			 * Get bootsize value for make_root_vdev().
1163 			 */
1164 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_BOOTSIZE) {
1165 				if (zfs_nicestrtonum(g_zfs, propval,
1166 				    &boot_size) < 0 || boot_size == 0) {
1167 					(void) fprintf(stderr,
1168 					    gettext("bad boot partition size "
1169 					    "'%s': %s\n"),  propval,
1170 					    libzfs_error_description(g_zfs));
1171 					goto errout;
1172 				}
1173 			}
1174 
1175 			/*
1176 			 * If the user is creating a pool that doesn't support
1177 			 * feature flags, don't enable any features.
1178 			 */
1179 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1180 				char *end;
1181 				u_longlong_t ver;
1182 
1183 				ver = strtoull(propval, &end, 10);
1184 				if (*end == '\0' &&
1185 				    ver < SPA_VERSION_FEATURES) {
1186 					enable_all_pool_feat = B_FALSE;
1187 				}
1188 			}
1189 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1190 				altroot = propval;
1191 			break;
1192 		case 'O':
1193 			if ((propval = strchr(optarg, '=')) == NULL) {
1194 				(void) fprintf(stderr, gettext("missing "
1195 				    "'=' for -O option\n"));
1196 				goto errout;
1197 			}
1198 			*propval = '\0';
1199 			propval++;
1200 
1201 			/*
1202 			 * Mountpoints are checked and then added later.
1203 			 * Uniquely among properties, they can be specified
1204 			 * more than once, to avoid conflict with -m.
1205 			 */
1206 			if (0 == strcmp(optarg,
1207 			    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1208 				mountpoint = propval;
1209 			} else if (add_prop_list(optarg, propval, &fsprops,
1210 			    B_FALSE)) {
1211 				goto errout;
1212 			}
1213 			break;
1214 		case 't':
1215 			/*
1216 			 * Sanity check temporary pool name.
1217 			 */
1218 			if (strchr(optarg, '/') != NULL) {
1219 				(void) fprintf(stderr, gettext("cannot create "
1220 				    "'%s': invalid character '/' in temporary "
1221 				    "name\n"), optarg);
1222 				(void) fprintf(stderr, gettext("use 'zfs "
1223 				    "create' to create a dataset\n"));
1224 				goto errout;
1225 			}
1226 
1227 			if (add_prop_list(zpool_prop_to_name(
1228 			    ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1229 				goto errout;
1230 			if (add_prop_list_default(zpool_prop_to_name(
1231 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1232 				goto errout;
1233 			tname = optarg;
1234 			break;
1235 		case ':':
1236 			(void) fprintf(stderr, gettext("missing argument for "
1237 			    "'%c' option\n"), optopt);
1238 			goto badusage;
1239 		case '?':
1240 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1241 			    optopt);
1242 			goto badusage;
1243 		}
1244 	}
1245 
1246 	argc -= optind;
1247 	argv += optind;
1248 
1249 	/* get pool name and check number of arguments */
1250 	if (argc < 1) {
1251 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
1252 		goto badusage;
1253 	}
1254 	if (argc < 2) {
1255 		(void) fprintf(stderr, gettext("missing vdev specification\n"));
1256 		goto badusage;
1257 	}
1258 
1259 	poolname = argv[0];
1260 
1261 	/*
1262 	 * As a special case, check for use of '/' in the name, and direct the
1263 	 * user to use 'zfs create' instead.
1264 	 */
1265 	if (strchr(poolname, '/') != NULL) {
1266 		(void) fprintf(stderr, gettext("cannot create '%s': invalid "
1267 		    "character '/' in pool name\n"), poolname);
1268 		(void) fprintf(stderr, gettext("use 'zfs create' to "
1269 		    "create a dataset\n"));
1270 		goto errout;
1271 	}
1272 
1273 	/*
1274 	 * Make sure the bootsize is set when ZPOOL_CREATE_BOOT_LABEL is used,
1275 	 * and not set otherwise.
1276 	 */
1277 	if (boot_type == ZPOOL_CREATE_BOOT_LABEL) {
1278 		const char *propname;
1279 		char *strptr, *buf = NULL;
1280 		int rv;
1281 
1282 		propname = zpool_prop_to_name(ZPOOL_PROP_BOOTSIZE);
1283 		if (nvlist_lookup_string(props, propname, &strptr) != 0) {
1284 			(void) asprintf(&buf, "%" PRIu64, boot_size);
1285 			if (buf == NULL) {
1286 				(void) fprintf(stderr,
1287 				    gettext("internal error: out of memory\n"));
1288 				goto errout;
1289 			}
1290 			rv = add_prop_list(propname, buf, &props, B_TRUE);
1291 			free(buf);
1292 			if (rv != 0)
1293 				goto errout;
1294 		}
1295 	} else {
1296 		const char *propname;
1297 		char *strptr;
1298 
1299 		propname = zpool_prop_to_name(ZPOOL_PROP_BOOTSIZE);
1300 		if (nvlist_lookup_string(props, propname, &strptr) == 0) {
1301 			(void) fprintf(stderr, gettext("error: setting boot "
1302 			    "partition size requires option '-B'\n"));
1303 			goto errout;
1304 		}
1305 	}
1306 
1307 	/* pass off to get_vdev_spec for bulk processing */
1308 	nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1309 	    boot_type, boot_size, argc - 1, argv + 1);
1310 	if (nvroot == NULL)
1311 		goto errout;
1312 
1313 	/* make_root_vdev() allows 0 toplevel children if there are spares */
1314 	if (!zfs_allocatable_devs(nvroot)) {
1315 		(void) fprintf(stderr, gettext("invalid vdev "
1316 		    "specification: at least one toplevel vdev must be "
1317 		    "specified\n"));
1318 		goto errout;
1319 	}
1320 
1321 	if (altroot != NULL && altroot[0] != '/') {
1322 		(void) fprintf(stderr, gettext("invalid alternate root '%s': "
1323 		    "must be an absolute path\n"), altroot);
1324 		goto errout;
1325 	}
1326 
1327 	/*
1328 	 * Check the validity of the mountpoint and direct the user to use the
1329 	 * '-m' mountpoint option if it looks like its in use.
1330 	 */
1331 	if (mountpoint == NULL ||
1332 	    (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1333 	    strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1334 		char buf[MAXPATHLEN];
1335 		DIR *dirp;
1336 
1337 		if (mountpoint && mountpoint[0] != '/') {
1338 			(void) fprintf(stderr, gettext("invalid mountpoint "
1339 			    "'%s': must be an absolute path, 'legacy', or "
1340 			    "'none'\n"), mountpoint);
1341 			goto errout;
1342 		}
1343 
1344 		if (mountpoint == NULL) {
1345 			if (altroot != NULL)
1346 				(void) snprintf(buf, sizeof (buf), "%s/%s",
1347 				    altroot, poolname);
1348 			else
1349 				(void) snprintf(buf, sizeof (buf), "/%s",
1350 				    poolname);
1351 		} else {
1352 			if (altroot != NULL)
1353 				(void) snprintf(buf, sizeof (buf), "%s%s",
1354 				    altroot, mountpoint);
1355 			else
1356 				(void) snprintf(buf, sizeof (buf), "%s",
1357 				    mountpoint);
1358 		}
1359 
1360 		if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1361 			(void) fprintf(stderr, gettext("mountpoint '%s' : "
1362 			    "%s\n"), buf, strerror(errno));
1363 			(void) fprintf(stderr, gettext("use '-m' "
1364 			    "option to provide a different default\n"));
1365 			goto errout;
1366 		} else if (dirp) {
1367 			int count = 0;
1368 
1369 			while (count < 3 && readdir(dirp) != NULL)
1370 				count++;
1371 			(void) closedir(dirp);
1372 
1373 			if (count > 2) {
1374 				(void) fprintf(stderr, gettext("mountpoint "
1375 				    "'%s' exists and is not empty\n"), buf);
1376 				(void) fprintf(stderr, gettext("use '-m' "
1377 				    "option to provide a "
1378 				    "different default\n"));
1379 				goto errout;
1380 			}
1381 		}
1382 	}
1383 
1384 	/*
1385 	 * Now that the mountpoint's validity has been checked, ensure that
1386 	 * the property is set appropriately prior to creating the pool.
1387 	 */
1388 	if (mountpoint != NULL) {
1389 		ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1390 		    mountpoint, &fsprops, B_FALSE);
1391 		if (ret != 0)
1392 			goto errout;
1393 	}
1394 
1395 	ret = 1;
1396 	if (dryrun) {
1397 		/*
1398 		 * For a dry run invocation, print out a basic message and run
1399 		 * through all the vdevs in the list and print out in an
1400 		 * appropriate hierarchy.
1401 		 */
1402 		(void) printf(gettext("would create '%s' with the "
1403 		    "following layout:\n\n"), poolname);
1404 
1405 		print_vdev_tree(NULL, poolname, nvroot, 0, "", 0);
1406 		print_vdev_tree(NULL, "dedup", nvroot, 0,
1407 		    VDEV_ALLOC_BIAS_DEDUP, 0);
1408 		print_vdev_tree(NULL, "special", nvroot, 0,
1409 		    VDEV_ALLOC_BIAS_SPECIAL, 0);
1410 		print_vdev_tree(NULL, "logs", nvroot, 0,
1411 		    VDEV_ALLOC_BIAS_LOG, 0);
1412 
1413 		ret = 0;
1414 	} else {
1415 		/*
1416 		 * Hand off to libzfs.
1417 		 */
1418 
1419 		spa_feature_t i;
1420 		for (i = 0; i < SPA_FEATURES; i++) {
1421 			char propname[MAXPATHLEN];
1422 			char *propval;
1423 			zfeature_info_t *feat = &spa_feature_table[i];
1424 			(void) snprintf(propname, sizeof (propname),
1425 			    "feature@%s", feat->fi_uname);
1426 
1427 			/*
1428 			 * Only features contained in props will be enabled:
1429 			 * remove from the nvlist every ZFS_FEATURE_DISABLED
1430 			 * value and add every missing ZFS_FEATURE_ENABLED if
1431 			 * enable_all_pool_feat is set.
1432 			 */
1433 			if (!nvlist_lookup_string(props, propname, &propval)) {
1434 				if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
1435 					(void) nvlist_remove_all(props,
1436 					    propname);
1437 			} else if (enable_all_pool_feat) {
1438 				ret = add_prop_list(propname,
1439 				    ZFS_FEATURE_ENABLED, &props, B_TRUE);
1440 				if (ret != 0)
1441 					goto errout;
1442 			}
1443 		}
1444 
1445 		ret = 1;
1446 		if (zpool_create(g_zfs, poolname,
1447 		    nvroot, props, fsprops) == 0) {
1448 			zfs_handle_t *pool = zfs_open(g_zfs,
1449 			    tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1450 			if (pool != NULL) {
1451 				if (zfs_mount(pool, NULL, 0) == 0)
1452 					ret = zfs_shareall(pool);
1453 				zfs_close(pool);
1454 			}
1455 		} else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1456 			(void) fprintf(stderr, gettext("pool name may have "
1457 			    "been omitted\n"));
1458 		}
1459 	}
1460 
1461 errout:
1462 	nvlist_free(nvroot);
1463 	nvlist_free(fsprops);
1464 	nvlist_free(props);
1465 	return (ret);
1466 badusage:
1467 	nvlist_free(fsprops);
1468 	nvlist_free(props);
1469 	usage(B_FALSE);
1470 	return (2);
1471 }
1472 
1473 /*
1474  * zpool destroy <pool>
1475  *
1476  *	-f	Forcefully unmount any datasets
1477  *
1478  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1479  */
1480 int
1481 zpool_do_destroy(int argc, char **argv)
1482 {
1483 	boolean_t force = B_FALSE;
1484 	int c;
1485 	char *pool;
1486 	zpool_handle_t *zhp;
1487 	int ret;
1488 
1489 	/* check options */
1490 	while ((c = getopt(argc, argv, "f")) != -1) {
1491 		switch (c) {
1492 		case 'f':
1493 			force = B_TRUE;
1494 			break;
1495 		case '?':
1496 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1497 			    optopt);
1498 			usage(B_FALSE);
1499 		}
1500 	}
1501 
1502 	argc -= optind;
1503 	argv += optind;
1504 
1505 	/* check arguments */
1506 	if (argc < 1) {
1507 		(void) fprintf(stderr, gettext("missing pool argument\n"));
1508 		usage(B_FALSE);
1509 	}
1510 	if (argc > 1) {
1511 		(void) fprintf(stderr, gettext("too many arguments\n"));
1512 		usage(B_FALSE);
1513 	}
1514 
1515 	pool = argv[0];
1516 
1517 	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1518 		/*
1519 		 * As a special case, check for use of '/' in the name, and
1520 		 * direct the user to use 'zfs destroy' instead.
1521 		 */
1522 		if (strchr(pool, '/') != NULL)
1523 			(void) fprintf(stderr, gettext("use 'zfs destroy' to "
1524 			    "destroy a dataset\n"));
1525 		return (1);
1526 	}
1527 
1528 	if (zpool_disable_datasets(zhp, force) != 0) {
1529 		(void) fprintf(stderr, gettext("could not destroy '%s': "
1530 		    "could not unmount datasets\n"), zpool_get_name(zhp));
1531 		return (1);
1532 	}
1533 
1534 	/* The history must be logged as part of the export */
1535 	log_history = B_FALSE;
1536 
1537 	ret = (zpool_destroy(zhp, history_str) != 0);
1538 
1539 	zpool_close(zhp);
1540 
1541 	return (ret);
1542 }
1543 
1544 /*
1545  * zpool export [-f] <pool> ...
1546  *
1547  *	-f	Forcefully unmount datasets
1548  *
1549  * Export the given pools.  By default, the command will attempt to cleanly
1550  * unmount any active datasets within the pool.  If the '-f' flag is specified,
1551  * then the datasets will be forcefully unmounted.
1552  */
1553 int
1554 zpool_do_export(int argc, char **argv)
1555 {
1556 	boolean_t force = B_FALSE;
1557 	boolean_t hardforce = B_FALSE;
1558 	int c;
1559 	zpool_handle_t *zhp;
1560 	int ret;
1561 	int i;
1562 
1563 	/* check options */
1564 	while ((c = getopt(argc, argv, "fF")) != -1) {
1565 		switch (c) {
1566 		case 'f':
1567 			force = B_TRUE;
1568 			break;
1569 		case 'F':
1570 			hardforce = B_TRUE;
1571 			break;
1572 		case '?':
1573 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
1574 			    optopt);
1575 			usage(B_FALSE);
1576 		}
1577 	}
1578 
1579 	argc -= optind;
1580 	argv += optind;
1581 
1582 	/* check arguments */
1583 	if (argc < 1) {
1584 		(void) fprintf(stderr, gettext("missing pool argument\n"));
1585 		usage(B_FALSE);
1586 	}
1587 
1588 	ret = 0;
1589 	for (i = 0; i < argc; i++) {
1590 		if ((zhp = zpool_open_canfail(g_zfs, argv[i])) == NULL) {
1591 			ret = 1;
1592 			continue;
1593 		}
1594 
1595 		if (zpool_disable_datasets(zhp, force) != 0) {
1596 			ret = 1;
1597 			zpool_close(zhp);
1598 			continue;
1599 		}
1600 
1601 		/* The history must be logged as part of the export */
1602 		log_history = B_FALSE;
1603 
1604 		if (hardforce) {
1605 			if (zpool_export_force(zhp, history_str) != 0)
1606 				ret = 1;
1607 		} else if (zpool_export(zhp, force, history_str) != 0) {
1608 			ret = 1;
1609 		}
1610 
1611 		zpool_close(zhp);
1612 	}
1613 
1614 	return (ret);
1615 }
1616 
1617 /*
1618  * Given a vdev configuration, determine the maximum width needed for the device
1619  * name column.
1620  */
1621 static int
1622 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
1623     int name_flags)
1624 {
1625 	char *name;
1626 	nvlist_t **child;
1627 	uint_t c, children;
1628 	int ret;
1629 
1630 	name = zpool_vdev_name(g_zfs, zhp, nv, name_flags | VDEV_NAME_TYPE_ID);
1631 	if (strlen(name) + depth > max)
1632 		max = strlen(name) + depth;
1633 
1634 	free(name);
1635 
1636 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1637 	    &child, &children) == 0) {
1638 		for (c = 0; c < children; c++)
1639 			if ((ret = max_width(zhp, child[c], depth + 2,
1640 			    max, name_flags)) > max)
1641 				max = ret;
1642 	}
1643 
1644 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1645 	    &child, &children) == 0) {
1646 		for (c = 0; c < children; c++)
1647 			if ((ret = max_width(zhp, child[c], depth + 2,
1648 			    max, name_flags)) > max)
1649 				max = ret;
1650 	}
1651 
1652 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1653 	    &child, &children) == 0) {
1654 		for (c = 0; c < children; c++)
1655 			if ((ret = max_width(zhp, child[c], depth + 2,
1656 			    max, name_flags)) > max)
1657 				max = ret;
1658 	}
1659 
1660 	return (max);
1661 }
1662 
1663 typedef struct spare_cbdata {
1664 	uint64_t	cb_guid;
1665 	zpool_handle_t	*cb_zhp;
1666 } spare_cbdata_t;
1667 
1668 static boolean_t
1669 find_vdev(nvlist_t *nv, uint64_t search)
1670 {
1671 	uint64_t guid;
1672 	nvlist_t **child;
1673 	uint_t c, children;
1674 
1675 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
1676 	    search == guid)
1677 		return (B_TRUE);
1678 
1679 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1680 	    &child, &children) == 0) {
1681 		for (c = 0; c < children; c++)
1682 			if (find_vdev(child[c], search))
1683 				return (B_TRUE);
1684 	}
1685 
1686 	return (B_FALSE);
1687 }
1688 
1689 static int
1690 find_spare(zpool_handle_t *zhp, void *data)
1691 {
1692 	spare_cbdata_t *cbp = data;
1693 	nvlist_t *config, *nvroot;
1694 
1695 	config = zpool_get_config(zhp, NULL);
1696 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1697 	    &nvroot) == 0);
1698 
1699 	if (find_vdev(nvroot, cbp->cb_guid)) {
1700 		cbp->cb_zhp = zhp;
1701 		return (1);
1702 	}
1703 
1704 	zpool_close(zhp);
1705 	return (0);
1706 }
1707 
1708 typedef struct status_cbdata {
1709 	int		cb_count;
1710 	int		cb_name_flags;
1711 	int		cb_namewidth;
1712 	boolean_t	cb_allpools;
1713 	boolean_t	cb_verbose;
1714 	boolean_t	cb_literal;
1715 	boolean_t	cb_explain;
1716 	boolean_t	cb_first;
1717 	boolean_t	cb_dedup_stats;
1718 	boolean_t	cb_print_status;
1719 	boolean_t	cb_print_slow_ios;
1720 	boolean_t	cb_print_vdev_init;
1721 	boolean_t	cb_print_vdev_trim;
1722 } status_cbdata_t;
1723 
1724 /*
1725  * Print vdev initialization status for leaves
1726  */
1727 static void
1728 print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
1729 {
1730 	if (verbose) {
1731 		if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE ||
1732 		    vs->vs_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
1733 		    vs->vs_initialize_state == VDEV_INITIALIZE_COMPLETE) &&
1734 		    !vs->vs_scan_removing) {
1735 			char zbuf[1024];
1736 			char tbuf[256];
1737 			struct tm zaction_ts;
1738 
1739 			time_t t = vs->vs_initialize_action_time;
1740 			int initialize_pct = 100;
1741 			if (vs->vs_initialize_state !=
1742 			    VDEV_INITIALIZE_COMPLETE) {
1743 				initialize_pct = (vs->vs_initialize_bytes_done *
1744 				    100 / (vs->vs_initialize_bytes_est + 1));
1745 			}
1746 
1747 			(void) localtime_r(&t, &zaction_ts);
1748 			(void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
1749 
1750 			switch (vs->vs_initialize_state) {
1751 			case VDEV_INITIALIZE_SUSPENDED:
1752 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1753 				    gettext("suspended, started at"), tbuf);
1754 				break;
1755 			case VDEV_INITIALIZE_ACTIVE:
1756 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1757 				    gettext("started at"), tbuf);
1758 				break;
1759 			case VDEV_INITIALIZE_COMPLETE:
1760 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1761 				    gettext("completed at"), tbuf);
1762 				break;
1763 			}
1764 
1765 			(void) printf(gettext("  (%d%% initialized%s)"),
1766 			    initialize_pct, zbuf);
1767 		} else {
1768 			(void) printf(gettext("  (uninitialized)"));
1769 		}
1770 	} else if (vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE) {
1771 		(void) printf(gettext("  (initializing)"));
1772 	}
1773 }
1774 
1775 /*
1776  * Print vdev TRIM status for leaves
1777  */
1778 static void
1779 print_status_trim(vdev_stat_t *vs, boolean_t verbose)
1780 {
1781 	if (verbose) {
1782 		if ((vs->vs_trim_state == VDEV_TRIM_ACTIVE ||
1783 		    vs->vs_trim_state == VDEV_TRIM_SUSPENDED ||
1784 		    vs->vs_trim_state == VDEV_TRIM_COMPLETE) &&
1785 		    !vs->vs_scan_removing) {
1786 			char zbuf[1024];
1787 			char tbuf[256];
1788 			struct tm zaction_ts;
1789 
1790 			time_t t = vs->vs_trim_action_time;
1791 			int trim_pct = 100;
1792 			if (vs->vs_trim_state != VDEV_TRIM_COMPLETE) {
1793 				trim_pct = (vs->vs_trim_bytes_done *
1794 				    100 / (vs->vs_trim_bytes_est + 1));
1795 			}
1796 
1797 			(void) localtime_r(&t, &zaction_ts);
1798 			(void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
1799 
1800 			switch (vs->vs_trim_state) {
1801 			case VDEV_TRIM_SUSPENDED:
1802 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1803 				    gettext("suspended, started at"), tbuf);
1804 				break;
1805 			case VDEV_TRIM_ACTIVE:
1806 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1807 				    gettext("started at"), tbuf);
1808 				break;
1809 			case VDEV_TRIM_COMPLETE:
1810 				(void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1811 				    gettext("completed at"), tbuf);
1812 				break;
1813 			}
1814 
1815 			(void) printf(gettext("  (%d%% trimmed%s)"),
1816 			    trim_pct, zbuf);
1817 		} else if (vs->vs_trim_notsup) {
1818 			(void) printf(gettext("  (trim unsupported)"));
1819 		} else {
1820 			(void) printf(gettext("  (untrimmed)"));
1821 		}
1822 	} else if (vs->vs_trim_state == VDEV_TRIM_ACTIVE) {
1823 		(void) printf(gettext("  (trimming)"));
1824 	}
1825 }
1826 
1827 /*
1828  * Print out configuration state as requested by status_callback.
1829  */
1830 static void
1831 print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
1832     nvlist_t *nv, int depth, boolean_t isspare)
1833 {
1834 	nvlist_t **child, *root;
1835 	uint_t c, children;
1836 	pool_scan_stat_t *ps = NULL;
1837 	vdev_stat_t *vs;
1838 	char rbuf[6], wbuf[6], cbuf[6];
1839 	char *vname;
1840 	uint64_t notpresent;
1841 	spare_cbdata_t spare_cb;
1842 	const char *state;
1843 	char *type;
1844 
1845 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1846 	    &child, &children) != 0)
1847 		children = 0;
1848 
1849 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1850 	    (uint64_t **)&vs, &c) == 0);
1851 
1852 	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1853 
1854 	if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
1855 		return;
1856 
1857 	state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
1858 	if (isspare) {
1859 		/*
1860 		 * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
1861 		 * online drives.
1862 		 */
1863 		if (vs->vs_aux == VDEV_AUX_SPARED)
1864 			state = "INUSE";
1865 		else if (vs->vs_state == VDEV_STATE_HEALTHY)
1866 			state = "AVAIL";
1867 	}
1868 
1869 	(void) printf("\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
1870 	    name, state);
1871 
1872 	if (!isspare) {
1873 		if (cb->cb_literal) {
1874 			printf(" %5llu %5llu %5llu",
1875 			    (u_longlong_t)vs->vs_read_errors,
1876 			    (u_longlong_t)vs->vs_write_errors,
1877 			    (u_longlong_t)vs->vs_checksum_errors);
1878 		} else {
1879 			zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
1880 			zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
1881 			zfs_nicenum(vs->vs_checksum_errors, cbuf,
1882 			    sizeof (cbuf));
1883 			printf(" %5s %5s %5s", rbuf, wbuf, cbuf);
1884 		}
1885 
1886 		if (cb->cb_print_slow_ios) {
1887 			if (children == 0)  {
1888 				/* Only leafs vdevs have slow IOs */
1889 				zfs_nicenum(vs->vs_slow_ios, rbuf,
1890 				    sizeof (rbuf));
1891 			} else {
1892 				(void) snprintf(rbuf, sizeof (rbuf), "-");
1893 			}
1894 
1895 			if (cb->cb_literal)
1896 				printf(" %5llu", (u_longlong_t)vs->vs_slow_ios);
1897 			else
1898 				printf(" %5s", rbuf);
1899 		}
1900 
1901 	}
1902 
1903 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
1904 	    &notpresent) == 0) {
1905 		char *path;
1906 		verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
1907 		(void) printf("  was %s", path);
1908 	} else if (vs->vs_aux != 0) {
1909 		(void) printf("  ");
1910 
1911 		switch (vs->vs_aux) {
1912 		case VDEV_AUX_OPEN_FAILED:
1913 			(void) printf(gettext("cannot open"));
1914 			break;
1915 
1916 		case VDEV_AUX_BAD_GUID_SUM:
1917 			(void) printf(gettext("missing device"));
1918 			break;
1919 
1920 		case VDEV_AUX_NO_REPLICAS:
1921 			(void) printf(gettext("insufficient replicas"));
1922 			break;
1923 
1924 		case VDEV_AUX_VERSION_NEWER:
1925 			(void) printf(gettext("newer version"));
1926 			break;
1927 
1928 		case VDEV_AUX_UNSUP_FEAT:
1929 			(void) printf(gettext("unsupported feature(s)"));
1930 			break;
1931 
1932 		case VDEV_AUX_SPARED:
1933 			verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
1934 			    &spare_cb.cb_guid) == 0);
1935 			if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
1936 				if (strcmp(zpool_get_name(spare_cb.cb_zhp),
1937 				    zpool_get_name(zhp)) == 0)
1938 					(void) printf(gettext("currently in "
1939 					    "use"));
1940 				else
1941 					(void) printf(gettext("in use by "
1942 					    "pool '%s'"),
1943 					    zpool_get_name(spare_cb.cb_zhp));
1944 				zpool_close(spare_cb.cb_zhp);
1945 			} else {
1946 				(void) printf(gettext("currently in use"));
1947 			}
1948 			break;
1949 
1950 		case VDEV_AUX_ERR_EXCEEDED:
1951 			(void) printf(gettext("too many errors"));
1952 			break;
1953 
1954 		case VDEV_AUX_IO_FAILURE:
1955 			(void) printf(gettext("experienced I/O failures"));
1956 			break;
1957 
1958 		case VDEV_AUX_BAD_LOG:
1959 			(void) printf(gettext("bad intent log"));
1960 			break;
1961 
1962 		case VDEV_AUX_EXTERNAL:
1963 			(void) printf(gettext("external device fault"));
1964 			break;
1965 
1966 		case VDEV_AUX_SPLIT_POOL:
1967 			(void) printf(gettext("split into new pool"));
1968 			break;
1969 
1970 		case VDEV_AUX_ACTIVE:
1971 			(void) printf(gettext("currently in use"));
1972 			break;
1973 
1974 		case VDEV_AUX_CHILDREN_OFFLINE:
1975 			(void) printf(gettext("all children offline"));
1976 			break;
1977 
1978 		default:
1979 			(void) printf(gettext("corrupted data"));
1980 			break;
1981 		}
1982 	}
1983 
1984 	/* The root vdev has the scrub/resilver stats */
1985 	root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
1986 	    ZPOOL_CONFIG_VDEV_TREE);
1987 	(void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS,
1988 	    (uint64_t **)&ps, &c);
1989 
1990 	if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0) {
1991 		if (vs->vs_scan_processed != 0) {
1992 			(void) printf(gettext("  (%s)"),
1993 			    (ps->pss_func == POOL_SCAN_RESILVER) ?
1994 			    "resilvering" : "repairing");
1995 		} else if (vs->vs_resilver_deferred) {
1996 			(void) printf(gettext("  (awaiting resilver)"));
1997 		}
1998 	}
1999 
2000 	/* Display vdev initialization and trim status for leaves */
2001 	if (children == 0) {
2002 		print_status_initialize(vs, cb->cb_print_vdev_init);
2003 		print_status_trim(vs, cb->cb_print_vdev_trim);
2004 	}
2005 
2006 	(void) printf("\n");
2007 
2008 	for (c = 0; c < children; c++) {
2009 		uint64_t islog = B_FALSE, ishole = B_FALSE;
2010 
2011 		/* Don't print logs or holes here */
2012 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2013 		    &islog);
2014 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
2015 		    &ishole);
2016 		if (islog || ishole)
2017 			continue;
2018 		/* Only print normal classes here */
2019 		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2020 			continue;
2021 
2022 		vname = zpool_vdev_name(g_zfs, zhp, child[c],
2023 		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2024 
2025 		print_status_config(zhp, cb, vname, child[c], depth + 2,
2026 		    isspare);
2027 		free(vname);
2028 	}
2029 }
2030 
2031 /*
2032  * Print the configuration of an exported pool.  Iterate over all vdevs in the
2033  * pool, printing out the name and status for each one.
2034  */
2035 static void
2036 print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
2037     int depth)
2038 {
2039 	nvlist_t **child;
2040 	uint_t c, children;
2041 	vdev_stat_t *vs;
2042 	char *type, *vname;
2043 
2044 	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2045 	if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
2046 	    strcmp(type, VDEV_TYPE_HOLE) == 0)
2047 		return;
2048 
2049 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2050 	    (uint64_t **)&vs, &c) == 0);
2051 
2052 	(void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
2053 	(void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
2054 
2055 	if (vs->vs_aux != 0) {
2056 		(void) printf("  ");
2057 
2058 		switch (vs->vs_aux) {
2059 		case VDEV_AUX_OPEN_FAILED:
2060 			(void) printf(gettext("cannot open"));
2061 			break;
2062 
2063 		case VDEV_AUX_BAD_GUID_SUM:
2064 			(void) printf(gettext("missing device"));
2065 			break;
2066 
2067 		case VDEV_AUX_NO_REPLICAS:
2068 			(void) printf(gettext("insufficient replicas"));
2069 			break;
2070 
2071 		case VDEV_AUX_VERSION_NEWER:
2072 			(void) printf(gettext("newer version"));
2073 			break;
2074 
2075 		case VDEV_AUX_UNSUP_FEAT:
2076 			(void) printf(gettext("unsupported feature(s)"));
2077 			break;
2078 
2079 		case VDEV_AUX_ERR_EXCEEDED:
2080 			(void) printf(gettext("too many errors"));
2081 			break;
2082 
2083 		case VDEV_AUX_ACTIVE:
2084 			(void) printf(gettext("currently in use"));
2085 			break;
2086 
2087 		case VDEV_AUX_CHILDREN_OFFLINE:
2088 			(void) printf(gettext("all children offline"));
2089 			break;
2090 
2091 		default:
2092 			(void) printf(gettext("corrupted data"));
2093 			break;
2094 		}
2095 	}
2096 	(void) printf("\n");
2097 
2098 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2099 	    &child, &children) != 0)
2100 		return;
2101 
2102 	for (c = 0; c < children; c++) {
2103 		uint64_t is_log = B_FALSE;
2104 
2105 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2106 		    &is_log);
2107 		if (is_log)
2108 			continue;
2109 		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2110 			continue;
2111 
2112 		vname = zpool_vdev_name(g_zfs, NULL, child[c],
2113 		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2114 		print_import_config(cb, vname, child[c], depth + 2);
2115 		free(vname);
2116 	}
2117 
2118 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
2119 	    &child, &children) == 0) {
2120 		(void) printf(gettext("\tcache\n"));
2121 		for (c = 0; c < children; c++) {
2122 			vname = zpool_vdev_name(g_zfs, NULL, child[c],
2123 			    cb->cb_name_flags);
2124 			(void) printf("\t  %s\n", vname);
2125 			free(vname);
2126 		}
2127 	}
2128 
2129 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
2130 	    &child, &children) == 0) {
2131 		(void) printf(gettext("\tspares\n"));
2132 		for (c = 0; c < children; c++) {
2133 			vname = zpool_vdev_name(g_zfs, NULL, child[c],
2134 			    cb->cb_name_flags);
2135 			(void) printf("\t  %s\n", vname);
2136 			free(vname);
2137 		}
2138 	}
2139 }
2140 
2141 /*
2142  * Print specialized class vdevs.
2143  *
2144  * These are recorded as top level vdevs in the main pool child array
2145  * but with "is_log" set to 1 or an "alloc_bias" string. We use either
2146  * print_status_config() or print_import_config() to print the top level
2147  * class vdevs then any of their children (eg mirrored slogs) are printed
2148  * recursively - which works because only the top level vdev is marked.
2149  */
2150 static void
2151 print_class_vdevs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv,
2152     const char *class)
2153 {
2154 	uint_t c, children;
2155 	nvlist_t **child;
2156 	boolean_t printed = B_FALSE;
2157 
2158 	assert(zhp != NULL || !cb->cb_verbose);
2159 
2160 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
2161 	    &children) != 0)
2162 		return;
2163 
2164 	for (c = 0; c < children; c++) {
2165 		uint64_t is_log = B_FALSE;
2166 		char *bias = NULL;
2167 		char *type = NULL;
2168 
2169 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2170 		    &is_log);
2171 
2172 		if (is_log) {
2173 			bias = VDEV_ALLOC_CLASS_LOGS;
2174 		} else {
2175 			(void) nvlist_lookup_string(child[c],
2176 			    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
2177 			(void) nvlist_lookup_string(child[c],
2178 			    ZPOOL_CONFIG_TYPE, &type);
2179 		}
2180 
2181 		if (bias == NULL || strcmp(bias, class) != 0)
2182 			continue;
2183 		if (!is_log && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2184 			continue;
2185 
2186 		if (!printed) {
2187 			(void) printf("\t%s\t\n", gettext(class));
2188 			printed = B_TRUE;
2189 		}
2190 
2191 		char *name = zpool_vdev_name(g_zfs, zhp, child[c],
2192 		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2193 		if (cb->cb_print_status)
2194 			print_status_config(zhp, cb, name, child[c], 2,
2195 			    B_FALSE);
2196 		else
2197 			print_import_config(cb, name, child[c], 2);
2198 		free(name);
2199 	}
2200 }
2201 
2202 /*
2203  * Display the status for the given pool.
2204  */
2205 static void
2206 show_import(nvlist_t *config)
2207 {
2208 	uint64_t pool_state;
2209 	vdev_stat_t *vs;
2210 	char *name;
2211 	uint64_t guid;
2212 	uint64_t hostid = 0;
2213 	char *msgid;
2214 	char *hostname = "unknown";
2215 	nvlist_t *nvroot, *nvinfo;
2216 	int reason;
2217 	zpool_errata_t errata;
2218 	const char *health;
2219 	uint_t vsc;
2220 	char *comment;
2221 	status_cbdata_t cb = { 0 };
2222 
2223 	verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2224 	    &name) == 0);
2225 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2226 	    &guid) == 0);
2227 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2228 	    &pool_state) == 0);
2229 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2230 	    &nvroot) == 0);
2231 
2232 	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2233 	    (uint64_t **)&vs, &vsc) == 0);
2234 	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2235 
2236 	reason = zpool_import_status(config, &msgid, &errata);
2237 
2238 	(void) printf(gettext("   pool: %s\n"), name);
2239 	(void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2240 	(void) printf(gettext("  state: %s"), health);
2241 	if (pool_state == POOL_STATE_DESTROYED)
2242 		(void) printf(gettext(" (DESTROYED)"));
2243 	(void) printf("\n");
2244 
2245 	switch (reason) {
2246 	case ZPOOL_STATUS_MISSING_DEV_R:
2247 	case ZPOOL_STATUS_MISSING_DEV_NR:
2248 	case ZPOOL_STATUS_BAD_GUID_SUM:
2249 		(void) printf(gettext(" status: One or more devices are "
2250 		    "missing from the system.\n"));
2251 		break;
2252 
2253 	case ZPOOL_STATUS_CORRUPT_LABEL_R:
2254 	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2255 		(void) printf(gettext(" status: One or more devices contains "
2256 		    "corrupted data.\n"));
2257 		break;
2258 
2259 	case ZPOOL_STATUS_CORRUPT_DATA:
2260 		(void) printf(
2261 		    gettext(" status: The pool data is corrupted.\n"));
2262 		break;
2263 
2264 	case ZPOOL_STATUS_OFFLINE_DEV:
2265 		(void) printf(gettext(" status: One or more devices "
2266 		    "are offlined.\n"));
2267 		break;
2268 
2269 	case ZPOOL_STATUS_CORRUPT_POOL:
2270 		(void) printf(gettext(" status: The pool metadata is "
2271 		    "corrupted.\n"));
2272 		break;
2273 
2274 	case ZPOOL_STATUS_VERSION_OLDER:
2275 		(void) printf(gettext(" status: The pool is formatted using a "
2276 		    "legacy on-disk version.\n"));
2277 		break;
2278 
2279 	case ZPOOL_STATUS_VERSION_NEWER:
2280 		(void) printf(gettext(" status: The pool is formatted using an "
2281 		    "incompatible version.\n"));
2282 		break;
2283 
2284 	case ZPOOL_STATUS_FEAT_DISABLED:
2285 		(void) printf(gettext(" status: Some supported features are "
2286 		    "not enabled on the pool.\n"));
2287 		break;
2288 
2289 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
2290 		(void) printf(gettext("status: The pool uses the following "
2291 		    "feature(s) not supported on this system:\n"));
2292 		zpool_print_unsup_feat(config);
2293 		break;
2294 
2295 	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2296 		(void) printf(gettext("status: The pool can only be accessed "
2297 		    "in read-only mode on this system. It\n\tcannot be "
2298 		    "accessed in read-write mode because it uses the "
2299 		    "following\n\tfeature(s) not supported on this system:\n"));
2300 		zpool_print_unsup_feat(config);
2301 		break;
2302 
2303 	case ZPOOL_STATUS_HOSTID_ACTIVE:
2304 		(void) printf(gettext(" status: The pool is currently "
2305 		    "imported by another system.\n"));
2306 		break;
2307 
2308 	case ZPOOL_STATUS_HOSTID_REQUIRED:
2309 		(void) printf(gettext(" status: The pool has the "
2310 		    "multihost property on.  It cannot\n\tbe safely imported "
2311 		    "when the system hostid is not set.\n"));
2312 		break;
2313 
2314 	case ZPOOL_STATUS_HOSTID_MISMATCH:
2315 		(void) printf(gettext(" status: The pool was last accessed by "
2316 		    "another system.\n"));
2317 		break;
2318 
2319 	case ZPOOL_STATUS_FAULTED_DEV_R:
2320 	case ZPOOL_STATUS_FAULTED_DEV_NR:
2321 		(void) printf(gettext(" status: One or more devices are "
2322 		    "faulted.\n"));
2323 		break;
2324 
2325 	case ZPOOL_STATUS_BAD_LOG:
2326 		(void) printf(gettext(" status: An intent log record cannot be "
2327 		    "read.\n"));
2328 		break;
2329 
2330 	case ZPOOL_STATUS_RESILVERING:
2331 		(void) printf(gettext(" status: One or more devices were being "
2332 		    "resilvered.\n"));
2333 		break;
2334 
2335 	case ZPOOL_STATUS_ERRATA:
2336 		(void) printf(gettext(" status: Errata #%d detected.\n"),
2337 		    errata);
2338 		break;
2339 
2340 	default:
2341 		/*
2342 		 * No other status can be seen when importing pools.
2343 		 */
2344 		assert(reason == ZPOOL_STATUS_OK);
2345 	}
2346 
2347 	/*
2348 	 * Print out an action according to the overall state of the pool.
2349 	 */
2350 	if (vs->vs_state == VDEV_STATE_HEALTHY) {
2351 		if (reason == ZPOOL_STATUS_VERSION_OLDER ||
2352 		    reason == ZPOOL_STATUS_FEAT_DISABLED) {
2353 			(void) printf(gettext(" action: The pool can be "
2354 			    "imported using its name or numeric identifier, "
2355 			    "though\n\tsome features will not be available "
2356 			    "without an explicit 'zpool upgrade'.\n"));
2357 		} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
2358 			(void) printf(gettext(" action: The pool can be "
2359 			    "imported using its name or numeric "
2360 			    "identifier and\n\tthe '-f' flag.\n"));
2361 		} else if (reason == ZPOOL_STATUS_ERRATA) {
2362 			switch (errata) {
2363 			case ZPOOL_ERRATA_NONE:
2364 				break;
2365 
2366 			case ZPOOL_ERRATA_ZOL_2094_SCRUB:
2367 				(void) printf(gettext(" action: The pool can "
2368 				    "be imported using its name or numeric "
2369 				    "identifier,\n\thowever there is a compat"
2370 				    "ibility issue which should be corrected"
2371 				    "\n\tby running 'zpool scrub'\n"));
2372 				break;
2373 
2374 			case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
2375 				(void) printf(gettext(" action: The pool can"
2376 				    "not be imported with this version of ZFS "
2377 				    "due to\n\tan active asynchronous destroy. "
2378 				    "Revert to an earlier version\n\tand "
2379 				    "allow the destroy to complete before "
2380 				    "updating.\n"));
2381 				break;
2382 
2383 			case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
2384 				(void) printf(gettext(" action: Existing "
2385 				    "encrypted datasets contain an on-disk "
2386 				    "incompatibility, which\n\tneeds to be "
2387 				    "corrected. Backup these datasets to new "
2388 				    "encrypted datasets\n\tand destroy the "
2389 				    "old ones.\n"));
2390 				break;
2391 			case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
2392 				(void) printf(gettext(" action: Any existing "
2393 				    "encrypted datasets contain an on-disk "
2394 				    "incompatibility which\n\tmay cause "
2395 				    "on-disk corruption with 'zfs recv' and "
2396 				    "which needs to be\n\tcorrected. Enable "
2397 				    "the bookmark_v2 feature and backup "
2398 				    "these datasets to new encrypted "
2399 				    "datasets and\n\tdestroy the old ones. "
2400 				    "If this pool does not contain any "
2401 				    "encrypted datasets, simply enable\n\t"
2402 				    "the bookmark_v2 feature.\n"));
2403 				break;
2404 			default:
2405 				/*
2406 				 * All errata must contain an action message.
2407 				 */
2408 				assert(0);
2409 			}
2410 		} else {
2411 			(void) printf(gettext(" action: The pool can be "
2412 			    "imported using its name or numeric "
2413 			    "identifier.\n"));
2414 		}
2415 	} else if (vs->vs_state == VDEV_STATE_DEGRADED) {
2416 		(void) printf(gettext(" action: The pool can be imported "
2417 		    "despite missing or damaged devices.  The\n\tfault "
2418 		    "tolerance of the pool may be compromised if imported.\n"));
2419 	} else {
2420 		switch (reason) {
2421 		case ZPOOL_STATUS_VERSION_NEWER:
2422 			(void) printf(gettext(" action: The pool cannot be "
2423 			    "imported.  Access the pool on a system running "
2424 			    "newer\n\tsoftware, or recreate the pool from "
2425 			    "backup.\n"));
2426 			break;
2427 		case ZPOOL_STATUS_UNSUP_FEAT_READ:
2428 			(void) printf(gettext("action: The pool cannot be "
2429 			    "imported. Access the pool on a system that "
2430 			    "supports\n\tthe required feature(s), or recreate "
2431 			    "the pool from backup.\n"));
2432 			break;
2433 		case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2434 			(void) printf(gettext("action: The pool cannot be "
2435 			    "imported in read-write mode. Import the pool "
2436 			    "with\n"
2437 			    "\t\"-o readonly=on\", access the pool on a system "
2438 			    "that supports the\n\trequired feature(s), or "
2439 			    "recreate the pool from backup.\n"));
2440 			break;
2441 		case ZPOOL_STATUS_MISSING_DEV_R:
2442 		case ZPOOL_STATUS_MISSING_DEV_NR:
2443 		case ZPOOL_STATUS_BAD_GUID_SUM:
2444 			(void) printf(gettext(" action: The pool cannot be "
2445 			    "imported. Attach the missing\n\tdevices and try "
2446 			    "again.\n"));
2447 			break;
2448 		case ZPOOL_STATUS_HOSTID_ACTIVE:
2449 			VERIFY0(nvlist_lookup_nvlist(config,
2450 			    ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
2451 
2452 			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2453 				hostname = fnvlist_lookup_string(nvinfo,
2454 				    ZPOOL_CONFIG_MMP_HOSTNAME);
2455 
2456 			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2457 				hostid = fnvlist_lookup_uint64(nvinfo,
2458 				    ZPOOL_CONFIG_MMP_HOSTID);
2459 
2460 			(void) printf(gettext(" action: The pool must be "
2461 			    "exported from %s (hostid=%lx)\n\tbefore it "
2462 			    "can be safely imported.\n"), hostname,
2463 			    (unsigned long) hostid);
2464 			break;
2465 		case ZPOOL_STATUS_HOSTID_REQUIRED:
2466 			(void) printf(gettext(" action: Check the SMF "
2467 			    "svc:/system/hostid service.\n"));
2468 			break;
2469 		default:
2470 			(void) printf(gettext(" action: The pool cannot be "
2471 			    "imported due to damaged devices or data.\n"));
2472 		}
2473 	}
2474 
2475 	/* Print the comment attached to the pool. */
2476 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
2477 		(void) printf(gettext("comment: %s\n"), comment);
2478 
2479 	/*
2480 	 * If the state is "closed" or "can't open", and the aux state
2481 	 * is "corrupt data":
2482 	 */
2483 	if (((vs->vs_state == VDEV_STATE_CLOSED) ||
2484 	    (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
2485 	    (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
2486 		if (pool_state == POOL_STATE_DESTROYED)
2487 			(void) printf(gettext("\tThe pool was destroyed, "
2488 			    "but can be imported using the '-Df' flags.\n"));
2489 		else if (pool_state != POOL_STATE_EXPORTED)
2490 			(void) printf(gettext("\tThe pool may be active on "
2491 			    "another system, but can be imported using\n\t"
2492 			    "the '-f' flag.\n"));
2493 	}
2494 
2495 	if (msgid != NULL)
2496 		(void) printf(gettext("   see: http://illumos.org/msg/%s\n"),
2497 		    msgid);
2498 
2499 	(void) printf(gettext(" config:\n\n"));
2500 
2501 	cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
2502 	    VDEV_NAME_TYPE_ID);
2503 	if (cb.cb_namewidth < 10)
2504 		cb.cb_namewidth = 10;
2505 
2506 	print_import_config(&cb, name, nvroot, 0);
2507 
2508 	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_DEDUP);
2509 	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
2510 	print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_CLASS_LOGS);
2511 
2512 	if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
2513 		(void) printf(gettext("\n\tAdditional devices are known to "
2514 		    "be part of this pool, though their\n\texact "
2515 		    "configuration cannot be determined.\n"));
2516 	}
2517 }
2518 
2519 static boolean_t
2520 zfs_force_import_required(nvlist_t *config)
2521 {
2522 	uint64_t state;
2523 	uint64_t hostid = 0;
2524 	nvlist_t *nvinfo;
2525 
2526 	state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
2527 	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
2528 
2529 	if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
2530 		return (B_TRUE);
2531 
2532 	nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2533 	if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
2534 		mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
2535 		    ZPOOL_CONFIG_MMP_STATE);
2536 
2537 		if (mmp_state != MMP_STATE_INACTIVE)
2538 			return (B_TRUE);
2539 	}
2540 
2541 	return (B_FALSE);
2542 }
2543 
2544 /*
2545  * Perform the import for the given configuration.  This passes the heavy
2546  * lifting off to zpool_import_props(), and then mounts the datasets contained
2547  * within the pool.
2548  */
2549 static int
2550 do_import(nvlist_t *config, const char *newname, const char *mntopts,
2551     nvlist_t *props, int flags)
2552 {
2553 	int ret = 0;
2554 	zpool_handle_t *zhp;
2555 	char *name;
2556 	uint64_t version;
2557 
2558 	name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
2559 	version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
2560 
2561 	if (!SPA_VERSION_IS_SUPPORTED(version)) {
2562 		(void) fprintf(stderr, gettext("cannot import '%s': pool "
2563 		    "is formatted using an unsupported ZFS version\n"), name);
2564 		return (1);
2565 	} else if (zfs_force_import_required(config) &&
2566 	    !(flags & ZFS_IMPORT_ANY_HOST)) {
2567 		mmp_state_t mmp_state = MMP_STATE_INACTIVE;
2568 		nvlist_t *nvinfo;
2569 
2570 		nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2571 		if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
2572 			mmp_state = fnvlist_lookup_uint64(nvinfo,
2573 			    ZPOOL_CONFIG_MMP_STATE);
2574 
2575 		if (mmp_state == MMP_STATE_ACTIVE) {
2576 			char *hostname = "<unknown>";
2577 			uint64_t hostid = 0;
2578 
2579 			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2580 				hostname = fnvlist_lookup_string(nvinfo,
2581 				    ZPOOL_CONFIG_MMP_HOSTNAME);
2582 
2583 			if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2584 				hostid = fnvlist_lookup_uint64(nvinfo,
2585 				    ZPOOL_CONFIG_MMP_HOSTID);
2586 
2587 			(void) fprintf(stderr, gettext("cannot import '%s': "
2588 			    "pool is imported on %s (hostid: "
2589 			    "0x%lx)\nExport the pool on the other system, "
2590 			    "then run 'zpool import'.\n"),
2591 			    name, hostname, (unsigned long) hostid);
2592 		} else if (mmp_state == MMP_STATE_NO_HOSTID) {
2593 			(void) fprintf(stderr, gettext("Cannot import '%s': "
2594 			    "pool has the multihost property on and the\n"
2595 			    "system's hostid is not set.\n"), name);
2596 		} else {
2597 			char *hostname = "<unknown>";
2598 			uint64_t timestamp = 0;
2599 			uint64_t hostid = 0;
2600 
2601 			if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
2602 				hostname = fnvlist_lookup_string(config,
2603 				    ZPOOL_CONFIG_HOSTNAME);
2604 
2605 			if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
2606 				timestamp = fnvlist_lookup_uint64(config,
2607 				    ZPOOL_CONFIG_TIMESTAMP);
2608 
2609 			if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
2610 				hostid = fnvlist_lookup_uint64(config,
2611 				    ZPOOL_CONFIG_HOSTID);
2612 
2613 			(void) fprintf(stderr, gettext("cannot import '%s': "
2614 			    "pool was previously in use from another system.\n"
2615 			    "Last accessed by %s (hostid=%lx) at %s"
2616 			    "The pool can be imported, use 'zpool import -f' "
2617 			    "to import the pool.\n"), name, hostname,
2618 			    (unsigned long)hostid, ctime((time_t *)&timestamp));
2619 
2620 		}
2621 
2622 		return (1);
2623 	}
2624 
2625 	if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
2626 		return (1);
2627 
2628 	if (newname != NULL)
2629 		name = (char *)newname;
2630 
2631 	if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
2632 		return (1);
2633 
2634 	/*
2635 	 * Loading keys is best effort. We don't want to return immediately
2636 	 * if it fails but we do want to give the error to the caller.
2637 	 */
2638 	if (flags & ZFS_IMPORT_LOAD_KEYS) {
2639 		ret = zfs_crypto_attempt_load_keys(g_zfs, name);
2640 		if (ret != 0)
2641 			ret = 1;
2642 	}
2643 
2644 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
2645 	    !(flags & ZFS_IMPORT_ONLY) &&
2646 	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
2647 		zpool_close(zhp);
2648 		return (1);
2649 	}
2650 
2651 	zpool_close(zhp);
2652 	return (ret);
2653 }
2654 
2655 typedef struct target_exists_args {
2656 	const char	*poolname;
2657 	uint64_t	poolguid;
2658 } target_exists_args_t;
2659 
2660 static int
2661 name_or_guid_exists(zpool_handle_t *zhp, void *data)
2662 {
2663 	target_exists_args_t *args = data;
2664 	nvlist_t *config = zpool_get_config(zhp, NULL);
2665 	int found = 0;
2666 
2667 	if (config == NULL)
2668 		return (0);
2669 
2670 	if (args->poolname != NULL) {
2671 		char *pool_name;
2672 
2673 		verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2674 		    &pool_name) == 0);
2675 		if (strcmp(pool_name, args->poolname) == 0)
2676 			found = 1;
2677 	} else {
2678 		uint64_t pool_guid;
2679 
2680 		verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2681 		    &pool_guid) == 0);
2682 		if (pool_guid == args->poolguid)
2683 			found = 1;
2684 	}
2685 	zpool_close(zhp);
2686 
2687 	return (found);
2688 }
2689 /*
2690  * zpool checkpoint <pool>
2691  *       checkpoint --discard <pool>
2692  *
2693  *	-d	Discard the checkpoint from a checkpointed
2694  *	--discard  pool.
2695  *
2696  * Checkpoints the specified pool, by taking a "snapshot" of its
2697  * current state. A pool can only have one checkpoint at a time.
2698  */
2699 int
2700 zpool_do_checkpoint(int argc, char **argv)
2701 {
2702 	boolean_t discard;
2703 	char *pool;
2704 	zpool_handle_t *zhp;
2705 	int c, err;
2706 
2707 	struct option long_options[] = {
2708 		{"discard", no_argument, NULL, 'd'},
2709 		{0, 0, 0, 0}
2710 	};
2711 
2712 	discard = B_FALSE;
2713 	while ((c = getopt_long(argc, argv, ":d", long_options, NULL)) != -1) {
2714 		switch (c) {
2715 		case 'd':
2716 			discard = B_TRUE;
2717 			break;
2718 		case '?':
2719 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
2720 			    optopt);
2721 			usage(B_FALSE);
2722 		}
2723 	}
2724 
2725 	argc -= optind;
2726 	argv += optind;
2727 
2728 	if (argc < 1) {
2729 		(void) fprintf(stderr, gettext("missing pool argument\n"));
2730 		usage(B_FALSE);
2731 	}
2732 
2733 	if (argc > 1) {
2734 		(void) fprintf(stderr, gettext("too many arguments\n"));
2735 		usage(B_FALSE);
2736 	}
2737 
2738 	pool = argv[0];
2739 
2740 	if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
2741 		/* As a special case, check for use of '/' in the name */
2742 		if (strchr(pool, '/') != NULL)
2743 			(void) fprintf(stderr, gettext("'zpool checkpoint' "
2744 			    "doesn't work on datasets. To save the state "
2745 			    "of a dataset from a specific point in time "
2746 			    "please use 'zfs snapshot'\n"));
2747 		return (1);
2748 	}
2749 
2750 	if (discard)
2751 		err = (zpool_discard_checkpoint(zhp) != 0);
2752 	else
2753 		err = (zpool_checkpoint(zhp) != 0);
2754 
2755 	zpool_close(zhp);
2756 
2757 	return (err);
2758 }
2759 
2760 #define	CHECKPOINT_OPT	1024
2761 
2762 /*
2763  * zpool import [-d dir] [-D]
2764  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
2765  *              [-d dir | -c cachefile] [-f] -a
2766  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
2767  *              [-d dir | -c cachefile] [-f] [-n] [-F] [-t]
2768  *              <pool | id> [newpool]
2769  *
2770  *	-c	Read pool information from a cachefile instead of searching
2771  *		devices.
2772  *
2773  *	-d	Scan in a specific directory, other than /dev/dsk.  More than
2774  *		one directory can be specified using multiple '-d' options.
2775  *
2776  *	-D	Scan for previously destroyed pools or import all or only
2777  *		specified destroyed pools.
2778  *
2779  *	-R	Temporarily import the pool, with all mountpoints relative to
2780  *		the given root.  The pool will remain exported when the machine
2781  *		is rebooted.
2782  *
2783  *	-V	Import even in the presence of faulted vdevs.  This is an
2784  *		intentionally undocumented option for testing purposes, and
2785  *		treats the pool configuration as complete, leaving any bad
2786  *		vdevs in the FAULTED state. In other words, it does verbatim
2787  *		import.
2788  *
2789  *	-f	Force import, even if it appears that the pool is active.
2790  *
2791  *	-F	Attempt rewind if necessary.
2792  *
2793  *	-n	See if rewind would work, but don't actually rewind.
2794  *
2795  *	-N	Import the pool but don't mount datasets.
2796  *
2797  *	-t	Use newpool as a temporary pool name instead of renaming
2798  *		the pool.
2799  *
2800  *	-T	Specify a starting txg to use for import. This option is
2801  *		intentionally undocumented option for testing purposes.
2802  *
2803  *	-a	Import all pools found.
2804  *
2805  *	-o	Set property=value and/or temporary mount options (without '=').
2806  *  -l	Load encryption keys while importing.
2807  *
2808  *	--rewind-to-checkpoint
2809  *		Import the pool and revert back to the checkpoint.
2810  *
2811  * The import command scans for pools to import, and import pools based on pool
2812  * name and GUID.  The pool can also be renamed as part of the import process.
2813  */
2814 int
2815 zpool_do_import(int argc, char **argv)
2816 {
2817 	char **searchdirs = NULL;
2818 	int nsearch = 0;
2819 	int c;
2820 	int err = 0;
2821 	nvlist_t *pools = NULL;
2822 	boolean_t do_all = B_FALSE;
2823 	boolean_t do_destroyed = B_FALSE;
2824 	char *mntopts = NULL;
2825 	nvpair_t *elem;
2826 	nvlist_t *config;
2827 	uint64_t searchguid = 0;
2828 	char *searchname = NULL;
2829 	char *propval;
2830 	nvlist_t *found_config;
2831 	nvlist_t *policy = NULL;
2832 	nvlist_t *props = NULL;
2833 	boolean_t first;
2834 	int flags = ZFS_IMPORT_NORMAL;
2835 	uint32_t rewind_policy = ZPOOL_NO_REWIND;
2836 	boolean_t dryrun = B_FALSE;
2837 	boolean_t do_rewind = B_FALSE;
2838 	boolean_t xtreme_rewind = B_FALSE;
2839 	boolean_t pool_exists = B_FALSE;
2840 	uint64_t pool_state, txg = -1ULL;
2841 	char *cachefile = NULL;
2842 	importargs_t idata = { 0 };
2843 	char *endptr;
2844 
2845 
2846 	struct option long_options[] = {
2847 		{"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
2848 		{0, 0, 0, 0}
2849 	};
2850 
2851 	/* check options */
2852 	while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:rR:tT:VX",
2853 	    long_options, NULL)) != -1) {
2854 		switch (c) {
2855 		case 'a':
2856 			do_all = B_TRUE;
2857 			break;
2858 		case 'c':
2859 			cachefile = optarg;
2860 			break;
2861 		case 'd':
2862 			if (searchdirs == NULL) {
2863 				searchdirs = safe_malloc(sizeof (char *));
2864 			} else {
2865 				char **tmp = safe_malloc((nsearch + 1) *
2866 				    sizeof (char *));
2867 				bcopy(searchdirs, tmp, nsearch *
2868 				    sizeof (char *));
2869 				free(searchdirs);
2870 				searchdirs = tmp;
2871 			}
2872 			searchdirs[nsearch++] = optarg;
2873 			break;
2874 		case 'D':
2875 			do_destroyed = B_TRUE;
2876 			break;
2877 		case 'f':
2878 			flags |= ZFS_IMPORT_ANY_HOST;
2879 			break;
2880 		case 'F':
2881 			do_rewind = B_TRUE;
2882 			break;
2883 		case 'l':
2884 			flags |= ZFS_IMPORT_LOAD_KEYS;
2885 			break;
2886 		case 'm':
2887 			flags |= ZFS_IMPORT_MISSING_LOG;
2888 			break;
2889 		case 'n':
2890 			dryrun = B_TRUE;
2891 			break;
2892 		case 'N':
2893 			flags |= ZFS_IMPORT_ONLY;
2894 			break;
2895 		case 'o':
2896 			if ((propval = strchr(optarg, '=')) != NULL) {
2897 				*propval = '\0';
2898 				propval++;
2899 				if (add_prop_list(optarg, propval,
2900 				    &props, B_TRUE))
2901 					goto error;
2902 			} else {
2903 				mntopts = optarg;
2904 			}
2905 			break;
2906 		case 'R':
2907 			if (add_prop_list(zpool_prop_to_name(
2908 			    ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
2909 				goto error;
2910 			if (add_prop_list_default(zpool_prop_to_name(
2911 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2912 				goto error;
2913 			break;
2914 		case 't':
2915 			flags |= ZFS_IMPORT_TEMP_NAME;
2916 			if (add_prop_list_default(zpool_prop_to_name(
2917 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2918 				goto error;
2919 			break;
2920 		case 'T':
2921 			errno = 0;
2922 			txg = strtoull(optarg, &endptr, 0);
2923 			if (errno != 0 || *endptr != '\0') {
2924 				(void) fprintf(stderr,
2925 				    gettext("invalid txg value\n"));
2926 				usage(B_FALSE);
2927 			}
2928 			rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
2929 			break;
2930 		case 'V':
2931 			flags |= ZFS_IMPORT_VERBATIM;
2932 			break;
2933 		case 'X':
2934 			xtreme_rewind = B_TRUE;
2935 			break;
2936 		case CHECKPOINT_OPT:
2937 			flags |= ZFS_IMPORT_CHECKPOINT;
2938 			break;
2939 		case ':':
2940 			(void) fprintf(stderr, gettext("missing argument for "
2941 			    "'%c' option\n"), optopt);
2942 			usage(B_FALSE);
2943 			break;
2944 		case '?':
2945 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
2946 			    optopt);
2947 			usage(B_FALSE);
2948 		}
2949 	}
2950 
2951 	argc -= optind;
2952 	argv += optind;
2953 
2954 	if (cachefile && nsearch != 0) {
2955 		(void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
2956 		usage(B_FALSE);
2957 	}
2958 
2959 	if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
2960 		(void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
2961 		usage(B_FALSE);
2962 	}
2963 
2964 	if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
2965 		(void) fprintf(stderr, gettext("-l is only meaningful during "
2966 		    "an import\n"));
2967 		usage(B_FALSE);
2968 	}
2969 
2970 	if ((dryrun || xtreme_rewind) && !do_rewind) {
2971 		(void) fprintf(stderr,
2972 		    gettext("-n or -X only meaningful with -F\n"));
2973 		usage(B_FALSE);
2974 	}
2975 	if (dryrun)
2976 		rewind_policy = ZPOOL_TRY_REWIND;
2977 	else if (do_rewind)
2978 		rewind_policy = ZPOOL_DO_REWIND;
2979 	if (xtreme_rewind)
2980 		rewind_policy |= ZPOOL_EXTREME_REWIND;
2981 
2982 	/* In the future, we can capture further policy and include it here */
2983 	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
2984 	    nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
2985 	    nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
2986 	    rewind_policy) != 0)
2987 		goto error;
2988 
2989 	if (searchdirs == NULL) {
2990 		searchdirs = safe_malloc(sizeof (char *));
2991 		searchdirs[0] = ZFS_DISK_ROOT;
2992 		nsearch = 1;
2993 	}
2994 
2995 	/* check argument count */
2996 	if (do_all) {
2997 		if (argc != 0) {
2998 			(void) fprintf(stderr, gettext("too many arguments\n"));
2999 			usage(B_FALSE);
3000 		}
3001 	} else {
3002 		if (argc > 2) {
3003 			(void) fprintf(stderr, gettext("too many arguments\n"));
3004 			usage(B_FALSE);
3005 		}
3006 
3007 		/*
3008 		 * Check for the SYS_CONFIG privilege.  We do this explicitly
3009 		 * here because otherwise any attempt to discover pools will
3010 		 * silently fail.
3011 		 */
3012 		if (argc == 0 && !priv_ineffect(PRIV_SYS_CONFIG)) {
3013 			(void) fprintf(stderr, gettext("cannot "
3014 			    "discover pools: permission denied\n"));
3015 			free(searchdirs);
3016 			nvlist_free(policy);
3017 			return (1);
3018 		}
3019 	}
3020 
3021 	/*
3022 	 * Depending on the arguments given, we do one of the following:
3023 	 *
3024 	 *	<none>	Iterate through all pools and display information about
3025 	 *		each one.
3026 	 *
3027 	 *	-a	Iterate through all pools and try to import each one.
3028 	 *
3029 	 *	<id>	Find the pool that corresponds to the given GUID/pool
3030 	 *		name and import that one.
3031 	 *
3032 	 *	-D	Above options applies only to destroyed pools.
3033 	 */
3034 	if (argc != 0) {
3035 		char *endptr;
3036 
3037 		errno = 0;
3038 		searchguid = strtoull(argv[0], &endptr, 10);
3039 		if (errno != 0 || *endptr != '\0') {
3040 			searchname = argv[0];
3041 			searchguid = 0;
3042 		}
3043 		found_config = NULL;
3044 
3045 		/*
3046 		 * User specified a name or guid.  Ensure it's unique.
3047 		 */
3048 		target_exists_args_t search = {searchname, searchguid};
3049 		pool_exists = zpool_iter(g_zfs, name_or_guid_exists, &search);
3050 	}
3051 
3052 	idata.path = searchdirs;
3053 	idata.paths = nsearch;
3054 	idata.poolname = searchname;
3055 	idata.guid = searchguid;
3056 	idata.cachefile = cachefile;
3057 	idata.policy = policy;
3058 
3059 	pools = zpool_search_import(g_zfs, &idata, &libzfs_config_ops);
3060 
3061 	if (pools != NULL && pool_exists &&
3062 	    (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
3063 		(void) fprintf(stderr, gettext("cannot import '%s': "
3064 		    "a pool with that name already exists\n"),
3065 		    argv[0]);
3066 		(void) fprintf(stderr, gettext("use the form 'zpool import "
3067 		    "[-t] <pool | id> <newpool>' to give it a new temporary "
3068 		    "or permanent name\n"));
3069 		err = 1;
3070 	} else if (pools == NULL && pool_exists) {
3071 		(void) fprintf(stderr, gettext("cannot import '%s': "
3072 		    "a pool with that name is already created/imported,\n"),
3073 		    argv[0]);
3074 		(void) fprintf(stderr, gettext("and no additional pools "
3075 		    "with that name were found\n"));
3076 		err = 1;
3077 	} else if (pools == NULL) {
3078 		if (argc != 0) {
3079 			(void) fprintf(stderr, gettext("cannot import '%s': "
3080 			    "no such pool available\n"), argv[0]);
3081 		}
3082 		err = 1;
3083 	}
3084 
3085 	if (err == 1) {
3086 		free(searchdirs);
3087 		nvlist_free(policy);
3088 		return (1);
3089 	}
3090 
3091 	/*
3092 	 * At this point we have a list of import candidate configs. Even if
3093 	 * we were searching by pool name or guid, we still need to
3094 	 * post-process the list to deal with pool state and possible
3095 	 * duplicate names.
3096 	 */
3097 	err = 0;
3098 	elem = NULL;
3099 	first = B_TRUE;
3100 	while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
3101 
3102 		verify(nvpair_value_nvlist(elem, &config) == 0);
3103 
3104 		verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
3105 		    &pool_state) == 0);
3106 		if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
3107 			continue;
3108 		if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
3109 			continue;
3110 
3111 		verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
3112 		    policy) == 0);
3113 
3114 		if (argc == 0) {
3115 			if (first)
3116 				first = B_FALSE;
3117 			else if (!do_all)
3118 				(void) printf("\n");
3119 
3120 			if (do_all) {
3121 				err |= do_import(config, NULL, mntopts,
3122 				    props, flags);
3123 			} else {
3124 				show_import(config);
3125 			}
3126 		} else if (searchname != NULL) {
3127 			char *name;
3128 
3129 			/*
3130 			 * We are searching for a pool based on name.
3131 			 */
3132 			verify(nvlist_lookup_string(config,
3133 			    ZPOOL_CONFIG_POOL_NAME, &name) == 0);
3134 
3135 			if (strcmp(name, searchname) == 0) {
3136 				if (found_config != NULL) {
3137 					(void) fprintf(stderr, gettext(
3138 					    "cannot import '%s': more than "
3139 					    "one matching pool\n"), searchname);
3140 					(void) fprintf(stderr, gettext(
3141 					    "import by numeric ID instead\n"));
3142 					err = B_TRUE;
3143 				}
3144 				found_config = config;
3145 			}
3146 		} else {
3147 			uint64_t guid;
3148 
3149 			/*
3150 			 * Search for a pool by guid.
3151 			 */
3152 			verify(nvlist_lookup_uint64(config,
3153 			    ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
3154 
3155 			if (guid == searchguid)
3156 				found_config = config;
3157 		}
3158 	}
3159 
3160 	/*
3161 	 * If we were searching for a specific pool, verify that we found a
3162 	 * pool, and then do the import.
3163 	 */
3164 	if (argc != 0 && err == 0) {
3165 		if (found_config == NULL) {
3166 			(void) fprintf(stderr, gettext("cannot import '%s': "
3167 			    "no such pool available\n"), argv[0]);
3168 			err = B_TRUE;
3169 		} else {
3170 			err |= do_import(found_config, argc == 1 ? NULL :
3171 			    argv[1], mntopts, props, flags);
3172 		}
3173 	}
3174 
3175 	/*
3176 	 * If we were just looking for pools, report an error if none were
3177 	 * found.
3178 	 */
3179 	if (argc == 0 && first)
3180 		(void) fprintf(stderr,
3181 		    gettext("no pools available to import\n"));
3182 
3183 error:
3184 	nvlist_free(props);
3185 	nvlist_free(pools);
3186 	nvlist_free(policy);
3187 	free(searchdirs);
3188 
3189 	return (err ? 1 : 0);
3190 }
3191 
3192 /*
3193  * zpool sync [-f] [pool] ...
3194  *
3195  * -f (undocumented) force uberblock (and config including zpool cache file)
3196  *    update.
3197  *
3198  * Sync the specified pool(s).
3199  * Without arguments "zpool sync" will sync all pools.
3200  * This command initiates TXG sync(s) and will return after the TXG(s) commit.
3201  *
3202  */
3203 static int
3204 zpool_do_sync(int argc, char **argv)
3205 {
3206 	int ret;
3207 	boolean_t force = B_FALSE;
3208 
3209 	/* check options */
3210 	while ((ret  = getopt(argc, argv, "f")) != -1) {
3211 		switch (ret) {
3212 		case 'f':
3213 			force = B_TRUE;
3214 			break;
3215 		case '?':
3216 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
3217 			    optopt);
3218 			usage(B_FALSE);
3219 		}
3220 	}
3221 
3222 	argc -= optind;
3223 	argv += optind;
3224 
3225 	/* if argc == 0 we will execute zpool_sync_one on all pools */
3226 	ret = for_each_pool(argc, argv, B_FALSE, NULL, zpool_sync_one, &force);
3227 
3228 	return (ret);
3229 }
3230 
3231 typedef struct iostat_cbdata {
3232 	uint64_t cb_flags;
3233 	int cb_name_flags;
3234 	int cb_namewidth;
3235 	int cb_iteration;
3236 	char **cb_vdev_names; /* Only show these vdevs */
3237 	unsigned int cb_vdev_names_count;
3238 	boolean_t cb_verbose;
3239 	boolean_t cb_literal;
3240 	boolean_t cb_scripted;
3241 	zpool_list_t *cb_list;
3242 } iostat_cbdata_t;
3243 
3244 /*  iostat labels */
3245 typedef struct name_and_columns {
3246 	const char *name;	/* Column name */
3247 	unsigned int columns;	/* Center name to this number of columns */
3248 } name_and_columns_t;
3249 
3250 #define	IOSTAT_MAX_LABELS	13	/* Max number of labels on one line */
3251 
3252 static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
3253 {
3254 	[IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
3255 	    {NULL}},
3256 	[IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3257 	    {"asyncq_wait", 2}, {"scrub"}, {"trim", 1}, {NULL}},
3258 	[IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
3259 	    {"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
3260 	    {"trimq_write", 2}, {NULL}},
3261 	[IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3262 	    {"asyncq_wait", 2}, {NULL}},
3263 	[IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
3264 	    {"async_read", 2}, {"async_write", 2}, {"scrub", 2},
3265 	    {"trim", 2}, {NULL}},
3266 
3267 };
3268 
3269 /* Shorthand - if "columns" field not set, default to 1 column */
3270 static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
3271 {
3272 	[IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
3273 	    {"write"}, {NULL}},
3274 	[IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3275 	    {"write"}, {"read"}, {"write"}, {"wait"}, {"wait"}, {NULL}},
3276 	[IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
3277 	    {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"},
3278 	    {"pend"}, {"activ"}, {NULL}},
3279 	[IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3280 	    {"write"}, {"read"}, {"write"}, {"scrub"}, {"trim"}, {NULL}},
3281 	[IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
3282 	    {"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"}, {NULL}},
3283 };
3284 
3285 static const char *histo_to_title[] = {
3286 	[IOS_L_HISTO] = "latency",
3287 	[IOS_RQ_HISTO] = "req_size",
3288 };
3289 
3290 /*
3291  * Return the number of labels in a null-terminated name_and_columns_t
3292  * array.
3293  *
3294  */
3295 static unsigned int
3296 label_array_len(const name_and_columns_t *labels)
3297 {
3298 	int i = 0;
3299 
3300 	while (labels[i].name)
3301 		i++;
3302 
3303 	return (i);
3304 }
3305 
3306 /*
3307  * Return the number of strings in a null-terminated string array.
3308  * For example:
3309  *
3310  *     const char foo[] = {"bar", "baz", NULL}
3311  *
3312  * returns 2
3313  */
3314 static uint64_t
3315 str_array_len(const char *array[])
3316 {
3317 	uint64_t i = 0;
3318 	while (array[i])
3319 		i++;
3320 
3321 	return (i);
3322 }
3323 
3324 
3325 /*
3326  * Return a default column width for default/latency/queue columns. This does
3327  * not include histograms, which have their columns autosized.
3328  */
3329 static unsigned int
3330 default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
3331 {
3332 	unsigned long column_width = 5; /* Normal niceprint */
3333 	static unsigned long widths[] = {
3334 		/*
3335 		 * Choose some sane default column sizes for printing the
3336 		 * raw numbers.
3337 		 */
3338 		[IOS_DEFAULT] = 15, /* 1PB capacity */
3339 		[IOS_LATENCY] = 10, /* 1B ns = 10sec */
3340 		[IOS_QUEUES] = 6,   /* 1M queue entries */
3341 		[IOS_L_HISTO] = 10, /* 1B ns = 10sec */
3342 		[IOS_RQ_HISTO] = 6, /* 1M queue entries */
3343 	};
3344 
3345 	if (cb->cb_literal)
3346 		column_width = widths[type];
3347 
3348 	return (column_width);
3349 }
3350 
3351 /*
3352  * Print the column labels, i.e:
3353  *
3354  *   capacity     operations     bandwidth
3355  * alloc   free   read  write   read  write  ...
3356  *
3357  * If force_column_width is set, use it for the column width.  If not set, use
3358  * the default column width.
3359  */
3360 void
3361 print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
3362     const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
3363 {
3364 	int i, idx, s;
3365 	int text_start, rw_column_width, spaces_to_end;
3366 	uint64_t flags = cb->cb_flags;
3367 	uint64_t f;
3368 	unsigned int column_width = force_column_width;
3369 
3370 	/* For each bit set in flags */
3371 	for (f = flags; f; f &= ~(1ULL << idx)) {
3372 		idx = lowbit64(f) - 1;
3373 		if (!force_column_width)
3374 			column_width = default_column_width(cb, idx);
3375 		/* Print our top labels centered over "read  write" label. */
3376 		for (i = 0; i < label_array_len(labels[idx]); i++) {
3377 			const char *name = labels[idx][i].name;
3378 			/*
3379 			 * We treat labels[][].columns == 0 as shorthand
3380 			 * for one column.  It makes writing out the label
3381 			 * tables more concise.
3382 			 */
3383 			unsigned int columns = MAX(1, labels[idx][i].columns);
3384 			unsigned int slen = strlen(name);
3385 
3386 			rw_column_width = (column_width * columns) +
3387 			    (2 * (columns - 1));
3388 
3389 			text_start = (int)((rw_column_width) / columns -
3390 			    slen / columns);
3391 			if (text_start < 0)
3392 				text_start = 0;
3393 
3394 			printf("  ");	/* Two spaces between columns */
3395 
3396 			/* Space from beginning of column to label */
3397 			for (s = 0; s < text_start; s++)
3398 				printf(" ");
3399 
3400 			printf("%s", name);
3401 
3402 			/* Print space after label to end of column */
3403 			spaces_to_end = rw_column_width - text_start - slen;
3404 			if (spaces_to_end < 0)
3405 				spaces_to_end = 0;
3406 
3407 			for (s = 0; s < spaces_to_end; s++)
3408 				printf(" ");
3409 		}
3410 	}
3411 }
3412 
3413 /*
3414  * Utility function to print out a line of dashes like:
3415  *
3416  *	--------------------------------  -----  -----  -----  -----  -----
3417  *
3418  * ...or a dashed named-row line like:
3419  *
3420  *	logs                                  -      -      -      -      -
3421  *
3422  * @cb:				iostat data
3423  *
3424  * @force_column_width		If non-zero, use the value as the column width.
3425  *				Otherwise use the default column widths.
3426  *
3427  * @name:			Print a dashed named-row line starting
3428  *				with @name.  Otherwise, print a regular
3429  *				dashed line.
3430  */
3431 static void
3432 print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
3433     const char *name)
3434 {
3435 	int i;
3436 	unsigned int namewidth;
3437 	uint64_t flags = cb->cb_flags;
3438 	uint64_t f;
3439 	int idx;
3440 	const name_and_columns_t *labels;
3441 	const char *title;
3442 
3443 
3444 	if (cb->cb_flags & IOS_ANYHISTO_M) {
3445 		title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3446 	} else if (cb->cb_vdev_names_count) {
3447 		title = "vdev";
3448 	} else  {
3449 		title = "pool";
3450 	}
3451 
3452 	namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3453 	    name ? strlen(name) : 0);
3454 
3455 
3456 	if (name) {
3457 		printf("%-*s", namewidth, name);
3458 	} else {
3459 		for (i = 0; i < namewidth; i++)
3460 			(void) printf("-");
3461 	}
3462 
3463 	/* For each bit in flags */
3464 	for (f = flags; f; f &= ~(1ULL << idx)) {
3465 		unsigned int column_width;
3466 		idx = lowbit64(f) - 1;
3467 		if (force_column_width)
3468 			column_width = force_column_width;
3469 		else
3470 			column_width = default_column_width(cb, idx);
3471 
3472 		labels = iostat_bottom_labels[idx];
3473 		for (i = 0; i < label_array_len(labels); i++) {
3474 			if (name)
3475 				printf("  %*s-", column_width - 1, " ");
3476 			else
3477 				printf("  %.*s", column_width,
3478 				    "--------------------");
3479 		}
3480 	}
3481 }
3482 
3483 
3484 static void
3485 print_iostat_separator_impl(iostat_cbdata_t *cb,
3486     unsigned int force_column_width)
3487 {
3488 	print_iostat_dashes(cb, force_column_width, NULL);
3489 }
3490 
3491 static void
3492 print_iostat_separator(iostat_cbdata_t *cb)
3493 {
3494 	print_iostat_separator_impl(cb, 0);
3495 }
3496 
3497 static void
3498 print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
3499     const char *histo_vdev_name)
3500 {
3501 	unsigned int namewidth;
3502 	const char *title;
3503 
3504 	if (cb->cb_flags & IOS_ANYHISTO_M) {
3505 		title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3506 	} else if (cb->cb_vdev_names_count) {
3507 		title = "vdev";
3508 	} else  {
3509 		title = "pool";
3510 	}
3511 
3512 	namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3513 	    histo_vdev_name ? strlen(histo_vdev_name) : 0);
3514 
3515 	if (histo_vdev_name)
3516 		printf("%-*s", namewidth, histo_vdev_name);
3517 	else
3518 		printf("%*s", namewidth, "");
3519 
3520 
3521 	print_iostat_labels(cb, force_column_width, iostat_top_labels);
3522 	printf("\n");
3523 
3524 	printf("%-*s", namewidth, title);
3525 
3526 	print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
3527 
3528 	printf("\n");
3529 
3530 	print_iostat_separator_impl(cb, force_column_width);
3531 
3532 	printf("\n");
3533 }
3534 
3535 static void
3536 print_iostat_header(iostat_cbdata_t *cb)
3537 {
3538 	print_iostat_header_impl(cb, 0, NULL);
3539 }
3540 
3541 /*
3542  * Display a single statistic.
3543  */
3544 static void
3545 print_one_stat(uint64_t value, enum zfs_nicenum_format format,
3546     unsigned int column_size, boolean_t scripted)
3547 {
3548 	char buf[64];
3549 
3550 	zfs_nicenum_format(value, buf, sizeof (buf), format);
3551 
3552 	if (scripted)
3553 		printf("\t%s", buf);
3554 	else
3555 		printf("  %*s", column_size, buf);
3556 }
3557 
3558 /*
3559  * Calculate the default vdev stats
3560  *
3561  * Subtract oldvs from newvs, apply a scaling factor, and save the resulting
3562  * stats into calcvs.
3563  */
3564 static void
3565 calc_default_iostats(vdev_stat_t *oldvs, vdev_stat_t *newvs,
3566     vdev_stat_t *calcvs)
3567 {
3568 	int i;
3569 
3570 	memcpy(calcvs, newvs, sizeof (*calcvs));
3571 	for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
3572 		calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);
3573 
3574 	for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
3575 		calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
3576 }
3577 
3578 /*
3579  * Internal representation of the extended iostats data.
3580  *
3581  * The extended iostat stats are exported in nvlists as either uint64_t arrays
3582  * or single uint64_t's.  We make both look like arrays to make them easier
3583  * to process.  In order to make single uint64_t's look like arrays, we set
3584  * __data to the stat data, and then set *data = &__data with count = 1.  Then,
3585  * we can just use *data and count.
3586  */
3587 struct stat_array {
3588 	uint64_t *data;
3589 	uint_t count;	/* Number of entries in data[] */
3590 	uint64_t __data; /* Only used when data is a single uint64_t */
3591 };
3592 
3593 static uint64_t
3594 stat_histo_max(struct stat_array *nva, unsigned int len)
3595 {
3596 	uint64_t max = 0;
3597 	int i;
3598 	for (i = 0; i < len; i++)
3599 		max = MAX(max, array64_max(nva[i].data, nva[i].count));
3600 
3601 	return (max);
3602 }
3603 
3604 /*
3605  * Helper function to lookup a uint64_t array or uint64_t value and store its
3606  * data as a stat_array.  If the nvpair is a single uint64_t value, then we make
3607  * it look like a one element array to make it easier to process.
3608  */
3609 static int
3610 nvpair64_to_stat_array(nvlist_t *nvl, const char *name,
3611     struct stat_array *nva)
3612 {
3613 	nvpair_t *tmp;
3614 	int ret;
3615 
3616 	verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
3617 	switch (nvpair_type(tmp)) {
3618 	case DATA_TYPE_UINT64_ARRAY:
3619 		ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
3620 		break;
3621 	case DATA_TYPE_UINT64:
3622 		ret = nvpair_value_uint64(tmp, &nva->__data);
3623 		nva->data = &nva->__data;
3624 		nva->count = 1;
3625 		break;
3626 	default:
3627 		/* Not a uint64_t */
3628 		ret = EINVAL;
3629 		break;
3630 	}
3631 
3632 	return (ret);
3633 }
3634 
3635 /*
3636  * Given a list of nvlist names, look up the extended stats in newnv and oldnv,
3637  * subtract them, and return the results in a newly allocated stat_array.
3638  * You must free the returned array after you are done with it with
3639  * free_calc_stats().
3640  *
3641  * Additionally, you can set "oldnv" to NULL if you simply want the newnv
3642  * values.
3643  */
3644 static struct stat_array *
3645 calc_and_alloc_stats_ex(const char **names, unsigned int len, nvlist_t *oldnv,
3646     nvlist_t *newnv)
3647 {
3648 	nvlist_t *oldnvx = NULL, *newnvx;
3649 	struct stat_array *oldnva, *newnva, *calcnva;
3650 	int i, j;
3651 	unsigned int alloc_size = (sizeof (struct stat_array)) * len;
3652 
3653 	/* Extract our extended stats nvlist from the main list */
3654 	verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
3655 	    &newnvx) == 0);
3656 	if (oldnv) {
3657 		verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
3658 		    &oldnvx) == 0);
3659 	}
3660 
3661 	newnva = safe_malloc(alloc_size);
3662 	oldnva = safe_malloc(alloc_size);
3663 	calcnva = safe_malloc(alloc_size);
3664 
3665 	for (j = 0; j < len; j++) {
3666 		verify(nvpair64_to_stat_array(newnvx, names[j],
3667 		    &newnva[j]) == 0);
3668 		calcnva[j].count = newnva[j].count;
3669 		alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
3670 		calcnva[j].data = safe_malloc(alloc_size);
3671 		memcpy(calcnva[j].data, newnva[j].data, alloc_size);
3672 
3673 		if (oldnvx) {
3674 			verify(nvpair64_to_stat_array(oldnvx, names[j],
3675 			    &oldnva[j]) == 0);
3676 			for (i = 0; i < oldnva[j].count; i++)
3677 				calcnva[j].data[i] -= oldnva[j].data[i];
3678 		}
3679 	}
3680 	free(newnva);
3681 	free(oldnva);
3682 	return (calcnva);
3683 }
3684 
3685 static void
3686 free_calc_stats(struct stat_array *nva, unsigned int len)
3687 {
3688 	int i;
3689 	for (i = 0; i < len; i++)
3690 		free(nva[i].data);
3691 
3692 	free(nva);
3693 }
3694 
3695 static void
3696 print_iostat_histo(struct stat_array *nva, unsigned int len,
3697     iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
3698     double scale)
3699 {
3700 	int i, j;
3701 	char buf[6];
3702 	uint64_t val;
3703 	enum zfs_nicenum_format format;
3704 	unsigned int buckets;
3705 	unsigned int start_bucket;
3706 
3707 	if (cb->cb_literal)
3708 		format = ZFS_NICENUM_RAW;
3709 	else
3710 		format = ZFS_NICENUM_1024;
3711 
3712 	/* All these histos are the same size, so just use nva[0].count */
3713 	buckets = nva[0].count;
3714 
3715 	if (cb->cb_flags & IOS_RQ_HISTO_M) {
3716 		/* Start at 512 - req size should never be lower than this */
3717 		start_bucket = 9;
3718 	} else {
3719 		start_bucket = 0;
3720 	}
3721 
3722 	for (j = start_bucket; j < buckets; j++) {
3723 		/* Print histogram bucket label */
3724 		if (cb->cb_flags & IOS_L_HISTO_M) {
3725 			/* Ending range of this bucket */
3726 			val = (1ULL << (j + 1)) - 1;
3727 			zfs_nicetime(val, buf, sizeof (buf));
3728 		} else {
3729 			/* Request size (starting range of bucket) */
3730 			val = (1UL << j);
3731 			zfs_nicenum(val, buf, sizeof (buf));
3732 		}
3733 
3734 		if (cb->cb_scripted)
3735 			printf("%llu", (u_longlong_t)val);
3736 		else
3737 			printf("%-*s", namewidth, buf);
3738 
3739 		/* Print the values on the line */
3740 		for (i = 0; i < len; i++) {
3741 			print_one_stat(nva[i].data[j] * scale, format,
3742 			    column_width, cb->cb_scripted);
3743 		}
3744 		printf("\n");
3745 	}
3746 }
3747 
3748 static void
3749 print_solid_separator(unsigned int length)
3750 {
3751 	while (length--)
3752 		printf("-");
3753 	printf("\n");
3754 }
3755 
3756 static void
3757 print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv,
3758     nvlist_t *newnv, double scale, const char *name)
3759 {
3760 	unsigned int column_width;
3761 	unsigned int namewidth;
3762 	unsigned int entire_width;
3763 	enum iostat_type type;
3764 	struct stat_array *nva;
3765 	const char **names;
3766 	unsigned int names_len;
3767 
3768 	/* What type of histo are we? */
3769 	type = IOS_HISTO_IDX(cb->cb_flags);
3770 
3771 	/* Get NULL-terminated array of nvlist names for our histo */
3772 	names = vsx_type_to_nvlist[type];
3773 	names_len = str_array_len(names); /* num of names */
3774 
3775 	nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);
3776 
3777 	if (cb->cb_literal) {
3778 		column_width = MAX(5,
3779 		    (unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
3780 	} else {
3781 		column_width = 5;
3782 	}
3783 
3784 	namewidth = MAX(cb->cb_namewidth,
3785 	    strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));
3786 
3787 	/*
3788 	 * Calculate the entire line width of what we're printing.  The
3789 	 * +2 is for the two spaces between columns:
3790 	 */
3791 	/*	 read  write				*/
3792 	/*	-----  -----				*/
3793 	/*	|___|  <---------- column_width		*/
3794 	/*						*/
3795 	/*	|__________|  <--- entire_width		*/
3796 	/*						*/
3797 	entire_width = namewidth + (column_width + 2) *
3798 	    label_array_len(iostat_bottom_labels[type]);
3799 
3800 	if (cb->cb_scripted)
3801 		printf("%s\n", name);
3802 	else
3803 		print_iostat_header_impl(cb, column_width, name);
3804 
3805 	print_iostat_histo(nva, names_len, cb, column_width,
3806 	    namewidth, scale);
3807 
3808 	free_calc_stats(nva, names_len);
3809 	if (!cb->cb_scripted)
3810 		print_solid_separator(entire_width);
3811 }
3812 
3813 /*
3814  * Calculate the average latency of a power-of-two latency histogram
3815  */
3816 static uint64_t
3817 single_histo_average(uint64_t *histo, unsigned int buckets)
3818 {
3819 	int i;
3820 	uint64_t count = 0, total = 0;
3821 
3822 	for (i = 0; i < buckets; i++) {
3823 		/*
3824 		 * Our buckets are power-of-two latency ranges.  Use the
3825 		 * midpoint latency of each bucket to calculate the average.
3826 		 * For example:
3827 		 *
3828 		 * Bucket	Midpoint
3829 		 * 8ns-15ns:	12ns
3830 		 * 16ns-31ns:	24ns
3831 		 * ...
3832 		 */
3833 		if (histo[i] != 0) {
3834 			total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
3835 			count += histo[i];
3836 		}
3837 	}
3838 
3839 	/* Prevent divide by zero */
3840 	return (count == 0 ? 0 : total / count);
3841 }
3842 
3843 static void
3844 print_iostat_queues(iostat_cbdata_t *cb, nvlist_t *oldnv,
3845     nvlist_t *newnv)
3846 {
3847 	int i;
3848 	uint64_t val;
3849 	const char *names[] = {
3850 		ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
3851 		ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
3852 		ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
3853 		ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
3854 		ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
3855 		ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
3856 		ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
3857 		ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
3858 		ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
3859 		ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
3860 		ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
3861 		ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
3862 	};
3863 
3864 	struct stat_array *nva;
3865 
3866 	unsigned int column_width = default_column_width(cb, IOS_QUEUES);
3867 	enum zfs_nicenum_format format;
3868 
3869 	nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);
3870 
3871 	if (cb->cb_literal)
3872 		format = ZFS_NICENUM_RAW;
3873 	else
3874 		format = ZFS_NICENUM_1024;
3875 
3876 	for (i = 0; i < ARRAY_SIZE(names); i++) {
3877 		val = nva[i].data[0];
3878 		print_one_stat(val, format, column_width, cb->cb_scripted);
3879 	}
3880 
3881 	free_calc_stats(nva, ARRAY_SIZE(names));
3882 }
3883 
3884 static void
3885 print_iostat_latency(iostat_cbdata_t *cb, nvlist_t *oldnv,
3886     nvlist_t *newnv)
3887 {
3888 	int i;
3889 	uint64_t val;
3890 	const char *names[] = {
3891 		ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
3892 		ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
3893 		ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
3894 		ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
3895 		ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
3896 		ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
3897 		ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
3898 		ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
3899 		ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
3900 		ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
3901 	};
3902 	struct stat_array *nva;
3903 
3904 	unsigned int column_width = default_column_width(cb, IOS_LATENCY);
3905 	enum zfs_nicenum_format format;
3906 
3907 	nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);
3908 
3909 	if (cb->cb_literal)
3910 		format = ZFS_NICENUM_RAWTIME;
3911 	else
3912 		format = ZFS_NICENUM_TIME;
3913 
3914 	/* Print our avg latencies on the line */
3915 	for (i = 0; i < ARRAY_SIZE(names); i++) {
3916 		/* Compute average latency for a latency histo */
3917 		val = single_histo_average(nva[i].data, nva[i].count);
3918 		print_one_stat(val, format, column_width, cb->cb_scripted);
3919 	}
3920 	free_calc_stats(nva, ARRAY_SIZE(names));
3921 }
3922 
3923 /*
3924  * Print default statistics (capacity/operations/bandwidth)
3925  */
3926 static void
3927 print_iostat_default(vdev_stat_t *vs, iostat_cbdata_t *cb, double scale)
3928 {
3929 	unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
3930 	enum zfs_nicenum_format format;
3931 	char na;	/* char to print for "not applicable" values */
3932 
3933 	if (cb->cb_literal) {
3934 		format = ZFS_NICENUM_RAW;
3935 		na = '0';
3936 	} else {
3937 		format = ZFS_NICENUM_1024;
3938 		na = '-';
3939 	}
3940 
3941 	/* only toplevel vdevs have capacity stats */
3942 	if (vs->vs_space == 0) {
3943 		if (cb->cb_scripted)
3944 			printf("\t%c\t%c", na, na);
3945 		else
3946 			printf("  %*c  %*c", column_width, na, column_width,
3947 			    na);
3948 	} else {
3949 		print_one_stat(vs->vs_alloc, format, column_width,
3950 		    cb->cb_scripted);
3951 		print_one_stat(vs->vs_space - vs->vs_alloc, format,
3952 		    column_width, cb->cb_scripted);
3953 	}
3954 
3955 	print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
3956 	    format, column_width, cb->cb_scripted);
3957 	print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
3958 	    format, column_width, cb->cb_scripted);
3959 	print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
3960 	    format, column_width, cb->cb_scripted);
3961 	print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
3962 	    format, column_width, cb->cb_scripted);
3963 }
3964 
3965 static const char *class_name[] = {
3966 	VDEV_ALLOC_BIAS_DEDUP,
3967 	VDEV_ALLOC_BIAS_SPECIAL,
3968 	VDEV_ALLOC_CLASS_LOGS
3969 };
3970 
3971 /*
3972  * Print out all the statistics for the given vdev.  This can either be the
3973  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
3974  * is a verbose output, and we don't want to display the toplevel pool stats.
3975  *
3976  * Returns the number of stat lines printed.
3977  */
3978 unsigned int
3979 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
3980     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
3981 {
3982 	nvlist_t **oldchild, **newchild;
3983 	uint_t c, children, oldchildren;
3984 	vdev_stat_t *oldvs, *newvs, *calcvs;
3985 	vdev_stat_t zerovs = { 0 };
3986 	char *vname;
3987 	int i;
3988 	int ret = 0;
3989 	uint64_t tdelta;
3990 	double scale;
3991 
3992 	if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
3993 		return (0);
3994 
3995 	calcvs = safe_malloc(sizeof (*calcvs));
3996 
3997 	if (oldnv != NULL) {
3998 		verify(nvlist_lookup_uint64_array(oldnv,
3999 		    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
4000 	} else {
4001 		oldvs = &zerovs;
4002 	}
4003 
4004 	/* Do we only want to see a specific vdev? */
4005 	for (i = 0; i < cb->cb_vdev_names_count; i++) {
4006 		/* Yes we do.  Is this the vdev? */
4007 		if (strcmp(name, cb->cb_vdev_names[i]) == 0) {
4008 			/*
4009 			 * This is our vdev.  Since it is the only vdev we
4010 			 * will be displaying, make depth = 0 so that it
4011 			 * doesn't get indented.
4012 			 */
4013 			depth = 0;
4014 			break;
4015 		}
4016 	}
4017 
4018 	if (cb->cb_vdev_names_count && (i == cb->cb_vdev_names_count)) {
4019 		/* Couldn't match the name */
4020 		goto children;
4021 	}
4022 
4023 
4024 	verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
4025 	    (uint64_t **)&newvs, &c) == 0);
4026 
4027 	/*
4028 	 * Print the vdev name unless it's is a histogram.  Histograms
4029 	 * display the vdev name in the header itself.
4030 	 */
4031 	if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
4032 		if (cb->cb_scripted) {
4033 			printf("%s", name);
4034 		} else {
4035 			if (strlen(name) + depth > cb->cb_namewidth)
4036 				(void) printf("%*s%s", depth, "", name);
4037 			else
4038 				(void) printf("%*s%s%*s", depth, "", name,
4039 				    (int)(cb->cb_namewidth - strlen(name) -
4040 				    depth), "");
4041 		}
4042 	}
4043 
4044 	/* Calculate our scaling factor */
4045 	tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
4046 	if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
4047 		/*
4048 		 * If we specify printing histograms with no time interval, then
4049 		 * print the histogram numbers over the entire lifetime of the
4050 		 * vdev.
4051 		 */
4052 		scale = 1;
4053 	} else {
4054 		if (tdelta == 0)
4055 			scale = 1.0;
4056 		else
4057 			scale = (double)NANOSEC / tdelta;
4058 	}
4059 
4060 	if (cb->cb_flags & IOS_DEFAULT_M) {
4061 		calc_default_iostats(oldvs, newvs, calcvs);
4062 		print_iostat_default(calcvs, cb, scale);
4063 	}
4064 	if (cb->cb_flags & IOS_LATENCY_M)
4065 		print_iostat_latency(cb, oldnv, newnv);
4066 	if (cb->cb_flags & IOS_QUEUES_M)
4067 		print_iostat_queues(cb, oldnv, newnv);
4068 	if (cb->cb_flags & IOS_ANYHISTO_M) {
4069 		printf("\n");
4070 		print_iostat_histos(cb, oldnv, newnv, scale, name);
4071 	}
4072 
4073 	if (!(cb->cb_flags & IOS_ANYHISTO_M))
4074 		printf("\n");
4075 
4076 	ret++;
4077 
4078 children:
4079 
4080 	free(calcvs);
4081 
4082 	if (!cb->cb_verbose)
4083 		return (ret);
4084 
4085 	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
4086 	    &newchild, &children) != 0)
4087 		return (ret);
4088 
4089 	if (oldnv) {
4090 		if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
4091 		    &oldchild, &oldchildren) != 0)
4092 			return (ret);
4093 
4094 		children = MIN(oldchildren, children);
4095 	}
4096 
4097 	/*
4098 	 * print normal top-level devices
4099 	 */
4100 	for (c = 0; c < children; c++) {
4101 		uint64_t ishole = B_FALSE, islog = B_FALSE;
4102 
4103 		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
4104 		    &ishole);
4105 
4106 		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
4107 		    &islog);
4108 
4109 		if (ishole || islog)
4110 			continue;
4111 
4112 		if (nvlist_exists(newchild[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
4113 			continue;
4114 
4115 		vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4116 		    cb->cb_name_flags);
4117 		ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
4118 		    newchild[c], cb, depth + 2);
4119 		free(vname);
4120 	}
4121 
4122 	/*
4123 	 * print all other top-level devices
4124 	 */
4125 	for (uint_t n = 0; n < 3; n++) {
4126 		boolean_t printed = B_FALSE;
4127 
4128 		for (c = 0; c < children; c++) {
4129 			uint64_t islog = B_FALSE;
4130 			char *bias = NULL;
4131 			char *type = NULL;
4132 
4133 			(void) nvlist_lookup_uint64(newchild[c],
4134 			    ZPOOL_CONFIG_IS_LOG, &islog);
4135 			if (islog) {
4136 				bias = VDEV_ALLOC_CLASS_LOGS;
4137 			} else {
4138 				(void) nvlist_lookup_string(newchild[c],
4139 				    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
4140 				(void) nvlist_lookup_string(newchild[c],
4141 				    ZPOOL_CONFIG_TYPE, &type);
4142 			}
4143 			if (bias == NULL || strcmp(bias, class_name[n]) != 0)
4144 				continue;
4145 			if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
4146 				continue;
4147 
4148 			if (!printed) {
4149 				if ((!(cb->cb_flags & IOS_ANYHISTO_M)) &&
4150 				    !cb->cb_scripted && !cb->cb_vdev_names) {
4151 					print_iostat_dashes(cb, 0,
4152 					    class_name[n]);
4153 				}
4154 				printf("\n");
4155 				printed = B_TRUE;
4156 			}
4157 
4158 			vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4159 			    cb->cb_name_flags);
4160 			ret += print_vdev_stats(zhp, vname, oldnv ?
4161 			    oldchild[c] : NULL, newchild[c], cb, depth + 2);
4162 			free(vname);
4163 		}
4164 
4165 	}
4166 
4167 	/*
4168 	 * Include level 2 ARC devices in iostat output
4169 	 */
4170 	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
4171 	    &newchild, &children) != 0)
4172 		return (ret);
4173 
4174 	if (oldnv) {
4175 		if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
4176 		    &oldchild, &oldchildren) != 0)
4177 			return (ret);
4178 
4179 		children = MIN(oldchildren, children);
4180 	}
4181 
4182 	if (children > 0) {
4183 		if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
4184 		    !cb->cb_vdev_names) {
4185 			print_iostat_dashes(cb, 0, "cache");
4186 		}
4187 		printf("\n");
4188 
4189 		for (c = 0; c < children; c++) {
4190 			vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4191 			    cb->cb_name_flags);
4192 			ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
4193 			    : NULL, newchild[c], cb, depth + 2);
4194 			free(vname);
4195 		}
4196 	}
4197 
4198 	return (ret);
4199 }
4200 
4201 static int
4202 refresh_iostat(zpool_handle_t *zhp, void *data)
4203 {
4204 	iostat_cbdata_t *cb = data;
4205 	boolean_t missing;
4206 
4207 	/*
4208 	 * If the pool has disappeared, remove it from the list and continue.
4209 	 */
4210 	if (zpool_refresh_stats(zhp, &missing) != 0)
4211 		return (-1);
4212 
4213 	if (missing)
4214 		pool_list_remove(cb->cb_list, zhp);
4215 
4216 	return (0);
4217 }
4218 
4219 /*
4220  * Callback to print out the iostats for the given pool.
4221  */
4222 int
4223 print_iostat(zpool_handle_t *zhp, void *data)
4224 {
4225 	iostat_cbdata_t *cb = data;
4226 	nvlist_t *oldconfig, *newconfig;
4227 	nvlist_t *oldnvroot, *newnvroot;
4228 	int ret;
4229 
4230 	newconfig = zpool_get_config(zhp, &oldconfig);
4231 
4232 	if (cb->cb_iteration == 1)
4233 		oldconfig = NULL;
4234 
4235 	verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
4236 	    &newnvroot) == 0);
4237 
4238 	if (oldconfig == NULL)
4239 		oldnvroot = NULL;
4240 	else
4241 		verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
4242 		    &oldnvroot) == 0);
4243 
4244 	ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
4245 	    cb, 0);
4246 	if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
4247 	    !cb->cb_scripted && cb->cb_verbose && !cb->cb_vdev_names_count) {
4248 		print_iostat_separator(cb);
4249 		printf("\n");
4250 	}
4251 
4252 	return (ret);
4253 }
4254 
4255 static int
4256 get_columns(void)
4257 {
4258 	struct winsize ws;
4259 	int columns = 80;
4260 	int error;
4261 
4262 	if (isatty(STDOUT_FILENO)) {
4263 		error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
4264 		if (error == 0)
4265 			columns = ws.ws_col;
4266 	} else {
4267 		columns = 999;
4268 	}
4269 
4270 	return (columns);
4271 }
4272 
4273 /*
4274  * Return the required length of the pool/vdev name column.  The minimum
4275  * allowed width and output formatting flags must be provided.
4276  */
4277 static int
4278 get_namewidth(zpool_handle_t *zhp, int min_width, int flags, boolean_t verbose)
4279 {
4280 	nvlist_t *config, *nvroot;
4281 	int width = min_width;
4282 
4283 	if ((config = zpool_get_config(zhp, NULL)) != NULL) {
4284 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4285 		    &nvroot) == 0);
4286 		unsigned int poolname_len = strlen(zpool_get_name(zhp));
4287 		if (verbose == B_FALSE) {
4288 			width = MAX(poolname_len, min_width);
4289 		} else {
4290 			width = MAX(poolname_len,
4291 			    max_width(zhp, nvroot, 0, min_width, flags));
4292 		}
4293 	}
4294 
4295 	return (width);
4296 }
4297 
4298 /*
4299  * Parse the input string, get the 'interval' and 'count' value if there is one.
4300  */
4301 static void
4302 get_interval_count(int *argcp, char **argv, float *iv,
4303     unsigned long *cnt)
4304 {
4305 	float interval = 0;
4306 	unsigned long count = 0;
4307 	int argc = *argcp, errno;
4308 
4309 	/*
4310 	 * Determine if the last argument is an integer or a pool name
4311 	 */
4312 	if (argc > 0 && isnumber(argv[argc - 1])) {
4313 		char *end;
4314 
4315 		errno = 0;
4316 		interval = strtof(argv[argc - 1], &end);
4317 
4318 		if (*end == '\0' && errno == 0) {
4319 			if (interval == 0) {
4320 				(void) fprintf(stderr, gettext("interval "
4321 				    "cannot be zero\n"));
4322 				usage(B_FALSE);
4323 			}
4324 			/*
4325 			 * Ignore the last parameter
4326 			 */
4327 			argc--;
4328 		} else {
4329 			/*
4330 			 * If this is not a valid number, just plow on.  The
4331 			 * user will get a more informative error message later
4332 			 * on.
4333 			 */
4334 			interval = 0;
4335 		}
4336 	}
4337 
4338 	/*
4339 	 * If the last argument is also an integer, then we have both a count
4340 	 * and an interval.
4341 	 */
4342 	if (argc > 0 && isnumber(argv[argc - 1])) {
4343 		char *end;
4344 
4345 		errno = 0;
4346 		count = interval;
4347 		interval = strtof(argv[argc - 1], &end);
4348 
4349 		if (*end == '\0' && errno == 0) {
4350 			if (interval == 0) {
4351 				(void) fprintf(stderr, gettext("interval "
4352 				    "cannot be zero\n"));
4353 				usage(B_FALSE);
4354 			}
4355 
4356 			/*
4357 			 * Ignore the last parameter
4358 			 */
4359 			argc--;
4360 		} else {
4361 			interval = 0;
4362 		}
4363 	}
4364 
4365 	*iv = interval;
4366 	*cnt = count;
4367 	*argcp = argc;
4368 }
4369 
4370 static void
4371 get_timestamp_arg(char c)
4372 {
4373 	if (c == 'u')
4374 		timestamp_fmt = UDATE;
4375 	else if (c == 'd')
4376 		timestamp_fmt = DDATE;
4377 	else
4378 		usage(B_FALSE);
4379 }
4380 
4381 /*
4382  * Return stat flags that are supported by all pools by both the module and
4383  * zpool iostat.  "*data" should be initialized to all 0xFFs before running.
4384  * It will get ANDed down until only the flags that are supported on all pools
4385  * remain.
4386  */
4387 static int
4388 get_stat_flags_cb(zpool_handle_t *zhp, void *data)
4389 {
4390 	uint64_t *mask = data;
4391 	nvlist_t *config, *nvroot, *nvx;
4392 	uint64_t flags = 0;
4393 	int i, j;
4394 
4395 	config = zpool_get_config(zhp, NULL);
4396 	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4397 	    &nvroot) == 0);
4398 
4399 	/* Default stats are always supported, but for completeness.. */
4400 	if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
4401 		flags |= IOS_DEFAULT_M;
4402 
4403 	/* Get our extended stats nvlist from the main list */
4404 	if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
4405 	    &nvx) != 0) {
4406 		/*
4407 		 * No extended stats; they're probably running an older
4408 		 * module.  No big deal, we support that too.
4409 		 */
4410 		goto end;
4411 	}
4412 
4413 	/* For each extended stat, make sure all its nvpairs are supported */
4414 	for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
4415 		if (!vsx_type_to_nvlist[j][0])
4416 			continue;
4417 
4418 		/* Start off by assuming the flag is supported, then check */
4419 		flags |= (1ULL << j);
4420 		for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
4421 			if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
4422 				/* flag isn't supported */
4423 				flags = flags & ~(1ULL  << j);
4424 				break;
4425 			}
4426 		}
4427 	}
4428 end:
4429 	*mask = *mask & flags;
4430 	return (0);
4431 }
4432 
4433 /*
4434  * Return a bitmask of stats that are supported on all pools by both the module
4435  * and zpool iostat.
4436  */
4437 static uint64_t
4438 get_stat_flags(zpool_list_t *list)
4439 {
4440 	uint64_t mask = -1;
4441 
4442 	/*
4443 	 * get_stat_flags_cb() will lop off bits from "mask" until only the
4444 	 * flags that are supported on all pools remain.
4445 	 */
4446 	(void) pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
4447 	return (mask);
4448 }
4449 
4450 /*
4451  * Return 1 if cb_data->cb_vdev_names[0] is this vdev's name, 0 otherwise.
4452  */
4453 static int
4454 is_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_data)
4455 {
4456 	iostat_cbdata_t *cb = cb_data;
4457 	char *name;
4458 
4459 	name = zpool_vdev_name(g_zfs, zhp, nv, cb->cb_name_flags);
4460 
4461 	if (strcmp(name, cb->cb_vdev_names[0]) == 0)
4462 		return (1); /* match */
4463 
4464 	return (0);
4465 }
4466 
4467 /*
4468  * Returns 1 if cb_data->cb_vdev_names[0] is a vdev name, 0 otherwise.
4469  */
4470 static int
4471 is_vdev(zpool_handle_t *zhp, void *cb_data)
4472 {
4473 	return (for_each_vdev(zhp, is_vdev_cb, cb_data));
4474 }
4475 
4476 /*
4477  * Check if vdevs are in a pool
4478  *
4479  * Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
4480  * return 0.  If pool_name is NULL, then search all pools.
4481  */
4482 static int
4483 are_vdevs_in_pool(int argc, char **argv, char *pool_name,
4484     iostat_cbdata_t *cb)
4485 {
4486 	char **tmp_name;
4487 	int ret = 0;
4488 	int i;
4489 	int pool_count = 0;
4490 
4491 	if ((argc == 0) || !*argv)
4492 		return (0);
4493 
4494 	if (pool_name)
4495 		pool_count = 1;
4496 
4497 	/* Temporarily hijack cb_vdev_names for a second... */
4498 	tmp_name = cb->cb_vdev_names;
4499 
4500 	/* Go though our list of prospective vdev names */
4501 	for (i = 0; i < argc; i++) {
4502 		cb->cb_vdev_names = argv + i;
4503 
4504 		/* Is this name a vdev in our pools? */
4505 		ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
4506 		    is_vdev, cb);
4507 		if (!ret) {
4508 			/* No match */
4509 			break;
4510 		}
4511 	}
4512 
4513 	cb->cb_vdev_names = tmp_name;
4514 
4515 	return (ret);
4516 }
4517 
4518 static int
4519 is_pool_cb(zpool_handle_t *zhp, void *data)
4520 {
4521 	char *name = data;
4522 	if (strcmp(name, zpool_get_name(zhp)) == 0)
4523 		return (1);
4524 
4525 	return (0);
4526 }
4527 
4528 /*
4529  * Do we have a pool named *name?  If so, return 1, otherwise 0.
4530  */
4531 static int
4532 is_pool(char *name)
4533 {
4534 	return (for_each_pool(0, NULL, B_TRUE, NULL,  is_pool_cb, name));
4535 }
4536 
4537 /* Are all our argv[] strings pool names?  If so return 1, 0 otherwise. */
4538 static int
4539 are_all_pools(int argc, char **argv)
4540 {
4541 	if ((argc == 0) || !*argv)
4542 		return (0);
4543 
4544 	while (--argc >= 0)
4545 		if (!is_pool(argv[argc]))
4546 			return (0);
4547 
4548 	return (1);
4549 }
4550 
4551 /*
4552  * Helper function to print out vdev/pool names we can't resolve.  Used for an
4553  * error message.
4554  */
4555 static void
4556 error_list_unresolved_vdevs(int argc, char **argv, char *pool_name,
4557     iostat_cbdata_t *cb)
4558 {
4559 	int i;
4560 	char *name;
4561 	char *str;
4562 	for (i = 0; i < argc; i++) {
4563 		name = argv[i];
4564 
4565 		if (is_pool(name))
4566 			str = gettext("pool");
4567 		else if (are_vdevs_in_pool(1, &name, pool_name, cb))
4568 			str = gettext("vdev in this pool");
4569 		else if (are_vdevs_in_pool(1, &name, NULL, cb))
4570 			str = gettext("vdev in another pool");
4571 		else
4572 			str = gettext("unknown");
4573 
4574 		fprintf(stderr, "\t%s (%s)\n", name, str);
4575 	}
4576 }
4577 
4578 /*
4579  * Same as get_interval_count(), but with additional checks to not misinterpret
4580  * guids as interval/count values.  Assumes VDEV_NAME_GUID is set in
4581  * cb.cb_name_flags.
4582  */
4583 static void
4584 get_interval_count_filter_guids(int *argc, char **argv, float *interval,
4585     unsigned long *count, iostat_cbdata_t *cb)
4586 {
4587 	char **tmpargv = argv;
4588 	int argc_for_interval = 0;
4589 
4590 	/* Is the last arg an interval value?  Or a guid? */
4591 	if (*argc >= 1 && !are_vdevs_in_pool(1, &argv[*argc - 1], NULL, cb)) {
4592 		/*
4593 		 * The last arg is not a guid, so it's probably an
4594 		 * interval value.
4595 		 */
4596 		argc_for_interval++;
4597 
4598 		if (*argc >= 2 &&
4599 		    !are_vdevs_in_pool(1, &argv[*argc - 2], NULL, cb)) {
4600 			/*
4601 			 * The 2nd to last arg is not a guid, so it's probably
4602 			 * an interval value.
4603 			 */
4604 			argc_for_interval++;
4605 		}
4606 	}
4607 
4608 	/* Point to our list of possible intervals */
4609 	tmpargv = &argv[*argc - argc_for_interval];
4610 
4611 	*argc = *argc - argc_for_interval;
4612 	get_interval_count(&argc_for_interval, tmpargv,
4613 	    interval, count);
4614 }
4615 
4616 /*
4617  * Floating point sleep().  Allows you to pass in a floating point value for
4618  * seconds.
4619  */
4620 static void
4621 fsleep(float sec)
4622 {
4623 	struct timespec req;
4624 	req.tv_sec = floor(sec);
4625 	req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
4626 	(void) nanosleep(&req, NULL);
4627 }
4628 
4629 /*
4630  * Set the minimum pool/vdev name column width.  The width must be at least 10,
4631  * but may be as large as the column width - 42 so it still fits on one line.
4632  */
4633 static int
4634 get_namewidth_iostat(zpool_handle_t *zhp, void *data)
4635 {
4636 	iostat_cbdata_t *cb = data;
4637 	int width, columns;
4638 
4639 	width = get_namewidth(zhp, cb->cb_namewidth, cb->cb_name_flags,
4640 	    cb->cb_verbose);
4641 	columns = get_columns();
4642 
4643 	if (width < 10)
4644 		width = 10;
4645 	if (width > columns - 42)
4646 		width = columns - 42;
4647 
4648 	cb->cb_namewidth = width;
4649 
4650 	return (0);
4651 }
4652 
4653 /*
4654  * zpool iostat [-ghHLpPvy] [[-lq]|[-r|-w]] [-n name] [-T d|u]
4655  *		[[ pool ...]|[pool vdev ...]|[vdev ...]]
4656  *		[interval [count]]
4657  *
4658  *	-g	Display guid for individual vdev name.
4659  *	-L	Follow links when resolving vdev path name.
4660  *	-P	Display full path for vdev name.
4661  *	-v	Display statistics for individual vdevs
4662  *	-h	Display help
4663  *	-p	Display values in parsable (exact) format.
4664  *	-H	Scripted mode.  Don't display headers, and separate properties
4665  *		by a single tab.
4666  *	-l	Display average latency
4667  *	-q	Display queue depths
4668  *	-w	Display latency histograms
4669  *	-r	Display request size histogram
4670  *	-T	Display a timestamp in date(1) or Unix format
4671  *	-n	Only print headers once
4672  *
4673  * This command can be tricky because we want to be able to deal with pool
4674  * creation/destruction as well as vdev configuration changes.  The bulk of this
4675  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
4676  * on pool_list_update() to detect the addition of new pools.  Configuration
4677  * changes are all handled within libzfs.
4678  */
4679 int
4680 zpool_do_iostat(int argc, char **argv)
4681 {
4682 	int c;
4683 	int ret;
4684 	int npools;
4685 	float interval = 0;
4686 	unsigned long count = 0;
4687 	int winheight = 24;
4688 	struct winsize win;
4689 	zpool_list_t *list;
4690 	boolean_t verbose = B_FALSE;
4691 	boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
4692 	boolean_t queues = B_FALSE, parseable = B_FALSE, scripted = B_FALSE;
4693 	boolean_t omit_since_boot = B_FALSE;
4694 	boolean_t guid = B_FALSE;
4695 	boolean_t follow_links = B_FALSE;
4696 	boolean_t full_name = B_FALSE;
4697 	boolean_t headers_once = B_FALSE;
4698 	iostat_cbdata_t cb = { 0 };
4699 
4700 	/* Used for printing error message */
4701 	const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
4702 	    [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};
4703 
4704 	uint64_t unsupported_flags;
4705 
4706 	/* check options */
4707 	while ((c = getopt(argc, argv, "gLPT:vyhplqrwnH")) != -1) {
4708 		switch (c) {
4709 		case 'g':
4710 			guid = B_TRUE;
4711 			break;
4712 		case 'L':
4713 			follow_links = B_TRUE;
4714 			break;
4715 		case 'P':
4716 			full_name = B_TRUE;
4717 			break;
4718 		case 'T':
4719 			get_timestamp_arg(*optarg);
4720 			break;
4721 		case 'v':
4722 			verbose = B_TRUE;
4723 			break;
4724 		case 'p':
4725 			parseable = B_TRUE;
4726 			break;
4727 		case 'l':
4728 			latency = B_TRUE;
4729 			break;
4730 		case 'q':
4731 			queues = B_TRUE;
4732 			break;
4733 		case 'H':
4734 			scripted = B_TRUE;
4735 			break;
4736 		case 'w':
4737 			l_histo = B_TRUE;
4738 			break;
4739 		case 'r':
4740 			rq_histo = B_TRUE;
4741 			break;
4742 		case 'y':
4743 			omit_since_boot = B_TRUE;
4744 			break;
4745 		case 'n':
4746 			headers_once = B_TRUE;
4747 			break;
4748 		case 'h':
4749 			usage(B_FALSE);
4750 			break;
4751 		case '?':
4752 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
4753 			    optopt);
4754 			usage(B_FALSE);
4755 		}
4756 	}
4757 
4758 	argc -= optind;
4759 	argv += optind;
4760 
4761 	cb.cb_literal = parseable;
4762 	cb.cb_scripted = scripted;
4763 
4764 	if (guid)
4765 		cb.cb_name_flags |= VDEV_NAME_GUID;
4766 	if (follow_links)
4767 		cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
4768 	if (full_name)
4769 		cb.cb_name_flags |= VDEV_NAME_PATH;
4770 	cb.cb_iteration = 0;
4771 	cb.cb_namewidth = 0;
4772 	cb.cb_verbose = verbose;
4773 
4774 	/* Get our interval and count values (if any) */
4775 	if (guid) {
4776 		get_interval_count_filter_guids(&argc, argv, &interval,
4777 		    &count, &cb);
4778 	} else {
4779 		get_interval_count(&argc, argv, &interval, &count);
4780 	}
4781 
4782 	if (argc == 0) {
4783 		/* No args, so just print the defaults. */
4784 	} else if (are_all_pools(argc, argv)) {
4785 		/* All the args are pool names */
4786 	} else if (are_vdevs_in_pool(argc, argv, NULL, &cb)) {
4787 		/* All the args are vdevs */
4788 		cb.cb_vdev_names = argv;
4789 		cb.cb_vdev_names_count = argc;
4790 		argc = 0; /* No pools to process */
4791 	} else if (are_all_pools(1, argv)) {
4792 		/* The first arg is a pool name */
4793 		if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0], &cb)) {
4794 			/* ...and the rest are vdev names */
4795 			cb.cb_vdev_names = argv + 1;
4796 			cb.cb_vdev_names_count = argc - 1;
4797 			argc = 1; /* One pool to process */
4798 		} else {
4799 			fprintf(stderr, gettext("Expected either a list of "));
4800 			fprintf(stderr, gettext("pools, or list of vdevs in"));
4801 			fprintf(stderr, " \"%s\", ", argv[0]);
4802 			fprintf(stderr, gettext("but got:\n"));
4803 			error_list_unresolved_vdevs(argc - 1, argv + 1,
4804 			    argv[0], &cb);
4805 			fprintf(stderr, "\n");
4806 			usage(B_FALSE);
4807 			return (1);
4808 		}
4809 	} else {
4810 		/*
4811 		 * The args don't make sense. The first arg isn't a pool name,
4812 		 * nor are all the args vdevs.
4813 		 */
4814 		fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
4815 		fprintf(stderr, "\n");
4816 		return (1);
4817 	}
4818 
4819 	if (cb.cb_vdev_names_count != 0) {
4820 		/*
4821 		 * If user specified vdevs, it implies verbose.
4822 		 */
4823 		cb.cb_verbose = B_TRUE;
4824 	}
4825 
4826 	/*
4827 	 * Construct the list of all interesting pools.
4828 	 */
4829 	ret = 0;
4830 	if ((list = pool_list_get(argc, argv, NULL, &ret)) == NULL)
4831 		return (1);
4832 
4833 	if (pool_list_count(list) == 0 && argc != 0) {
4834 		pool_list_free(list);
4835 		return (1);
4836 	}
4837 
4838 	if (pool_list_count(list) == 0 && interval == 0) {
4839 		pool_list_free(list);
4840 		(void) fprintf(stderr, gettext("no pools available\n"));
4841 		return (1);
4842 	}
4843 
4844 	if ((l_histo || rq_histo) && (queues || latency)) {
4845 		pool_list_free(list);
4846 		(void) fprintf(stderr,
4847 		    gettext("[-r|-w] isn't allowed with [-q|-l]\n"));
4848 		usage(B_FALSE);
4849 		return (1);
4850 	}
4851 
4852 	if (l_histo && rq_histo) {
4853 		pool_list_free(list);
4854 		(void) fprintf(stderr,
4855 		    gettext("Only one of [-r|-w] can be passed at a time\n"));
4856 		usage(B_FALSE);
4857 		return (1);
4858 	}
4859 
4860 	/*
4861 	 * Enter the main iostat loop.
4862 	 */
4863 	cb.cb_list = list;
4864 
4865 	if (l_histo) {
4866 		/*
4867 		 * Histograms tables look out of place when you try to display
4868 		 * them with the other stats, so make a rule that you can only
4869 		 * print histograms by themselves.
4870 		 */
4871 		cb.cb_flags = IOS_L_HISTO_M;
4872 	} else if (rq_histo) {
4873 		cb.cb_flags = IOS_RQ_HISTO_M;
4874 	} else {
4875 		cb.cb_flags = IOS_DEFAULT_M;
4876 		if (latency)
4877 			cb.cb_flags |= IOS_LATENCY_M;
4878 		if (queues)
4879 			cb.cb_flags |= IOS_QUEUES_M;
4880 	}
4881 
4882 	/*
4883 	 * See if the module supports all the stats we want to display.
4884 	 */
4885 	unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
4886 	if (unsupported_flags) {
4887 		uint64_t f;
4888 		int idx;
4889 		fprintf(stderr,
4890 		    gettext("The loaded zfs module doesn't support:"));
4891 
4892 		/* for each bit set in unsupported_flags */
4893 		for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
4894 			idx = lowbit64(f) - 1;
4895 			fprintf(stderr, " -%c", flag_to_arg[idx]);
4896 		}
4897 
4898 		fprintf(stderr, ".  Try running a newer module.\n"),
4899 		    pool_list_free(list);
4900 
4901 		return (1);
4902 	}
4903 
4904 	for (;;) {
4905 		if ((npools = pool_list_count(list)) == 0)
4906 			(void) fprintf(stderr, gettext("no pools available\n"));
4907 		else {
4908 			/*
4909 			 * If this is the first iteration and -y was supplied
4910 			 * we skip any printing.
4911 			 */
4912 			boolean_t skip = (omit_since_boot &&
4913 			    cb.cb_iteration == 0);
4914 
4915 			/*
4916 			 * Refresh all statistics.  This is done as an
4917 			 * explicit step before calculating the maximum name
4918 			 * width, so that any configuration changes are
4919 			 * properly accounted for.
4920 			 */
4921 			(void) pool_list_iter(list, B_FALSE, refresh_iostat,
4922 			    &cb);
4923 
4924 			/*
4925 			 * Iterate over all pools to determine the maximum width
4926 			 * for the pool / device name column across all pools.
4927 			 */
4928 			cb.cb_namewidth = 0;
4929 			(void) pool_list_iter(list, B_FALSE,
4930 			    get_namewidth_iostat, &cb);
4931 
4932 			if (timestamp_fmt != NODATE)
4933 				print_timestamp(timestamp_fmt);
4934 
4935 			/*
4936 			 * Check terminal size so we can print headers
4937 			 * even when terminal window has its height
4938 			 * changed.
4939 			 */
4940 			if (headers_once == B_FALSE) {
4941 				if (ioctl(1, TIOCGWINSZ, &win) != -1) {
4942 					if (win.ws_row <= 0) {
4943 						headers_once = B_TRUE;
4944 					} else {
4945 						winheight = win.ws_row;
4946 					}
4947 				}
4948 			}
4949 			/*
4950 			 * Are we connected to TTY? If not, headers_once
4951 			 * should be true, to avoid breaking scripts.
4952 			 */
4953 			if (isatty(fileno(stdout)) == 0)
4954 				headers_once = B_TRUE;
4955 
4956 			/*
4957 			 * If it's the first time and we're not skipping it,
4958 			 * or either skip or verbose mode, print the header.
4959 			 *
4960 			 * The histogram code explicitly prints its header on
4961 			 * every vdev, so skip this for histograms.
4962 			 */
4963 			if (((++cb.cb_iteration == 1 && !skip) ||
4964 			    (skip != verbose) ||
4965 			    (!headers_once &&
4966 			    (cb.cb_iteration % winheight) == 0)) &&
4967 			    (!(cb.cb_flags & IOS_ANYHISTO_M)) &&
4968 			    !cb.cb_scripted)
4969 				print_iostat_header(&cb);
4970 
4971 			if (skip) {
4972 				(void) fsleep(interval);
4973 				continue;
4974 			}
4975 
4976 			(void) pool_list_iter(list, B_FALSE, print_iostat, &cb);
4977 
4978 			/*
4979 			 * If there's more than one pool, and we're not in
4980 			 * verbose mode (which prints a separator for us),
4981 			 * then print a separator.
4982 			 *
4983 			 * In addition, if we're printing specific vdevs then
4984 			 * we also want an ending separator.
4985 			 */
4986 			if (((npools > 1 && !verbose &&
4987 			    !(cb.cb_flags & IOS_ANYHISTO_M)) ||
4988 			    (!(cb.cb_flags & IOS_ANYHISTO_M) &&
4989 			    cb.cb_vdev_names_count)) &&
4990 			    !cb.cb_scripted) {
4991 				print_iostat_separator(&cb);
4992 				printf("\n");
4993 			}
4994 		}
4995 
4996 		/*
4997 		 * Flush the output so that redirection to a file isn't buffered
4998 		 * indefinitely.
4999 		 */
5000 		(void) fflush(stdout);
5001 
5002 		if (interval == 0)
5003 			break;
5004 
5005 		if (count != 0 && --count == 0)
5006 			break;
5007 
5008 		(void) fsleep(interval);
5009 	}
5010 
5011 	pool_list_free(list);
5012 
5013 	return (ret);
5014 }
5015 
5016 typedef struct list_cbdata {
5017 	boolean_t	cb_verbose;
5018 	int		cb_name_flags;
5019 	int		cb_namewidth;
5020 	boolean_t	cb_scripted;
5021 	zprop_list_t	*cb_proplist;
5022 	boolean_t	cb_literal;
5023 } list_cbdata_t;
5024 
5025 
5026 /*
5027  * Given a list of columns to display, output appropriate headers for each one.
5028  */
5029 static void
5030 print_header(list_cbdata_t *cb)
5031 {
5032 	zprop_list_t *pl = cb->cb_proplist;
5033 	char headerbuf[ZPOOL_MAXPROPLEN];
5034 	const char *header;
5035 	boolean_t first = B_TRUE;
5036 	boolean_t right_justify;
5037 	size_t width = 0;
5038 
5039 	for (; pl != NULL; pl = pl->pl_next) {
5040 		width = pl->pl_width;
5041 		if (first && cb->cb_verbose) {
5042 			/*
5043 			 * Reset the width to accommodate the verbose listing
5044 			 * of devices.
5045 			 */
5046 			width = cb->cb_namewidth;
5047 		}
5048 
5049 		if (!first)
5050 			(void) printf("  ");
5051 		else
5052 			first = B_FALSE;
5053 
5054 		right_justify = B_FALSE;
5055 		if (pl->pl_prop != ZPROP_INVAL) {
5056 			header = zpool_prop_column_name(pl->pl_prop);
5057 			right_justify = zpool_prop_align_right(pl->pl_prop);
5058 		} else {
5059 			int i;
5060 
5061 			for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
5062 				headerbuf[i] = toupper(pl->pl_user_prop[i]);
5063 			headerbuf[i] = '\0';
5064 			header = headerbuf;
5065 		}
5066 
5067 		if (pl->pl_next == NULL && !right_justify)
5068 			(void) printf("%s", header);
5069 		else if (right_justify)
5070 			(void) printf("%*s", width, header);
5071 		else
5072 			(void) printf("%-*s", width, header);
5073 
5074 	}
5075 
5076 	(void) printf("\n");
5077 }
5078 
5079 /*
5080  * Given a pool and a list of properties, print out all the properties according
5081  * to the described layout. Used by zpool_do_list().
5082  */
5083 static void
5084 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
5085 {
5086 	zprop_list_t *pl = cb->cb_proplist;
5087 	boolean_t first = B_TRUE;
5088 	char property[ZPOOL_MAXPROPLEN];
5089 	char *propstr;
5090 	boolean_t right_justify;
5091 	size_t width;
5092 
5093 	for (; pl != NULL; pl = pl->pl_next) {
5094 
5095 		width = pl->pl_width;
5096 		if (first && cb->cb_verbose) {
5097 			/*
5098 			 * Reset the width to accommodate the verbose listing
5099 			 * of devices.
5100 			 */
5101 			width = cb->cb_namewidth;
5102 		}
5103 
5104 		if (!first) {
5105 			if (cb->cb_scripted)
5106 				(void) printf("\t");
5107 			else
5108 				(void) printf("  ");
5109 		} else {
5110 			first = B_FALSE;
5111 		}
5112 
5113 		right_justify = B_FALSE;
5114 		if (pl->pl_prop != ZPROP_INVAL) {
5115 			if (zpool_get_prop(zhp, pl->pl_prop, property,
5116 			    sizeof (property), NULL, cb->cb_literal) != 0)
5117 				propstr = "-";
5118 			else
5119 				propstr = property;
5120 
5121 			right_justify = zpool_prop_align_right(pl->pl_prop);
5122 		} else if ((zpool_prop_feature(pl->pl_user_prop) ||
5123 		    zpool_prop_unsupported(pl->pl_user_prop)) &&
5124 		    zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
5125 		    sizeof (property)) == 0) {
5126 			propstr = property;
5127 		} else {
5128 			propstr = "-";
5129 		}
5130 
5131 
5132 		/*
5133 		 * If this is being called in scripted mode, or if this is the
5134 		 * last column and it is left-justified, don't include a width
5135 		 * format specifier.
5136 		 */
5137 		if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
5138 			(void) printf("%s", propstr);
5139 		else if (right_justify)
5140 			(void) printf("%*s", width, propstr);
5141 		else
5142 			(void) printf("%-*s", width, propstr);
5143 	}
5144 
5145 	(void) printf("\n");
5146 }
5147 
5148 static void
5149 print_one_column(zpool_prop_t prop, uint64_t value, boolean_t scripted,
5150     boolean_t valid)
5151 {
5152 	char propval[64];
5153 	boolean_t fixed;
5154 	size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
5155 
5156 	switch (prop) {
5157 	case ZPOOL_PROP_EXPANDSZ:
5158 	case ZPOOL_PROP_CHECKPOINT:
5159 		if (value == 0)
5160 			(void) strlcpy(propval, "-", sizeof (propval));
5161 		else
5162 			zfs_nicenum(value, propval, sizeof (propval));
5163 		break;
5164 	case ZPOOL_PROP_FRAGMENTATION:
5165 		if (value == ZFS_FRAG_INVALID) {
5166 			(void) strlcpy(propval, "-", sizeof (propval));
5167 		} else {
5168 			(void) snprintf(propval, sizeof (propval), "%llu%%",
5169 			    value);
5170 		}
5171 		break;
5172 	case ZPOOL_PROP_CAPACITY:
5173 		(void) snprintf(propval, sizeof (propval),
5174 		    value < 1000 ? "%1.2f%%" : value < 10000 ?
5175 		    "%2.1f%%" : "%3.0f%%", value / 100.0);
5176 		break;
5177 	default:
5178 		zfs_nicenum(value, propval, sizeof (propval));
5179 	}
5180 
5181 	if (!valid)
5182 		(void) strlcpy(propval, "-", sizeof (propval));
5183 
5184 	if (scripted)
5185 		(void) printf("\t%s", propval);
5186 	else
5187 		(void) printf("  %*s", width, propval);
5188 }
5189 
5190 /*
5191  * print static default line per vdev
5192  */
5193 void
5194 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
5195     list_cbdata_t *cb, int depth)
5196 {
5197 	nvlist_t **child;
5198 	vdev_stat_t *vs;
5199 	uint_t c, children;
5200 	char *vname;
5201 	boolean_t scripted = cb->cb_scripted;
5202 	uint64_t islog = B_FALSE;
5203 	char *dashes = "%-*s      -      -      -         -      -      -\n";
5204 
5205 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
5206 	    (uint64_t **)&vs, &c) == 0);
5207 
5208 	if (name != NULL) {
5209 		boolean_t toplevel = (vs->vs_space != 0);
5210 		uint64_t cap;
5211 
5212 		if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
5213 			return;
5214 
5215 		if (scripted)
5216 			(void) printf("\t%s", name);
5217 		else if (strlen(name) + depth > cb->cb_namewidth)
5218 			(void) printf("%*s%s", depth, "", name);
5219 		else
5220 			(void) printf("%*s%s%*s", depth, "", name,
5221 			    (int)(cb->cb_namewidth - strlen(name) - depth), "");
5222 
5223 		/*
5224 		 * Print the properties for the individual vdevs. Some
5225 		 * properties are only applicable to toplevel vdevs. The
5226 		 * 'toplevel' boolean value is passed to the print_one_column()
5227 		 * to indicate that the value is valid.
5228 		 */
5229 		print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, scripted,
5230 		    toplevel);
5231 		print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, scripted,
5232 		    toplevel);
5233 		print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
5234 		    scripted, toplevel);
5235 		print_one_column(ZPOOL_PROP_CHECKPOINT,
5236 		    vs->vs_checkpoint_space, scripted, toplevel);
5237 		print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, scripted,
5238 		    B_TRUE);
5239 		print_one_column(ZPOOL_PROP_FRAGMENTATION,
5240 		    vs->vs_fragmentation, scripted,
5241 		    (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel));
5242 		cap = (vs->vs_space == 0) ? 0 :
5243 		    (vs->vs_alloc * 10000 / vs->vs_space);
5244 		print_one_column(ZPOOL_PROP_CAPACITY, cap, scripted, toplevel);
5245 		(void) printf("\n");
5246 	}
5247 
5248 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
5249 	    &child, &children) != 0)
5250 		return;
5251 
5252 	/* list the normal vdevs first */
5253 	for (c = 0; c < children; c++) {
5254 		uint64_t ishole = B_FALSE;
5255 
5256 		if (nvlist_lookup_uint64(child[c],
5257 		    ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
5258 			continue;
5259 
5260 		if (nvlist_lookup_uint64(child[c],
5261 		    ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog)
5262 			continue;
5263 
5264 		if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
5265 			continue;
5266 
5267 		vname = zpool_vdev_name(g_zfs, zhp, child[c],
5268 		    cb->cb_name_flags);
5269 		print_list_stats(zhp, vname, child[c], cb, depth + 2);
5270 		free(vname);
5271 	}
5272 
5273 	/* list the classes: 'logs', 'dedup', and 'special' */
5274 	for (uint_t n = 0; n < 3; n++) {
5275 		boolean_t printed = B_FALSE;
5276 
5277 		for (c = 0; c < children; c++) {
5278 			char *bias = NULL;
5279 			char *type = NULL;
5280 
5281 			if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
5282 			    &islog) == 0 && islog) {
5283 				bias = VDEV_ALLOC_CLASS_LOGS;
5284 			} else {
5285 				(void) nvlist_lookup_string(child[c],
5286 				    ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
5287 				(void) nvlist_lookup_string(child[c],
5288 				    ZPOOL_CONFIG_TYPE, &type);
5289 			}
5290 			if (bias == NULL || strcmp(bias, class_name[n]) != 0)
5291 				continue;
5292 			if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
5293 				continue;
5294 
5295 			if (!printed) {
5296 				/* LINTED E_SEC_PRINTF_VAR_FMT */
5297 				(void) printf(dashes, cb->cb_namewidth,
5298 				    class_name[n]);
5299 				printed = B_TRUE;
5300 			}
5301 			vname = zpool_vdev_name(g_zfs, zhp, child[c],
5302 			    cb->cb_name_flags);
5303 			print_list_stats(zhp, vname, child[c], cb, depth + 2);
5304 			free(vname);
5305 		}
5306 	}
5307 
5308 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
5309 	    &child, &children) == 0 && children > 0) {
5310 		/* LINTED E_SEC_PRINTF_VAR_FMT */
5311 		(void) printf(dashes, cb->cb_namewidth, "cache");
5312 		for (c = 0; c < children; c++) {
5313 			vname = zpool_vdev_name(g_zfs, zhp, child[c],
5314 			    cb->cb_name_flags);
5315 			print_list_stats(zhp, vname, child[c], cb, depth + 2);
5316 			free(vname);
5317 		}
5318 	}
5319 
5320 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
5321 	    &children) == 0 && children > 0) {
5322 		/* LINTED E_SEC_PRINTF_VAR_FMT */
5323 		(void) printf(dashes, cb->cb_namewidth, "spare");
5324 		for (c = 0; c < children; c++) {
5325 			vname = zpool_vdev_name(g_zfs, zhp, child[c],
5326 			    cb->cb_name_flags);
5327 			print_list_stats(zhp, vname, child[c], cb, depth + 2);
5328 			free(vname);
5329 		}
5330 	}
5331 }
5332 
5333 /*
5334  * Generic callback function to list a pool.
5335  */
5336 int
5337 list_callback(zpool_handle_t *zhp, void *data)
5338 {
5339 	list_cbdata_t *cbp = data;
5340 	nvlist_t *config;
5341 	nvlist_t *nvroot;
5342 
5343 	config = zpool_get_config(zhp, NULL);
5344 
5345 	if (cbp->cb_verbose) {
5346 		config = zpool_get_config(zhp, NULL);
5347 
5348 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5349 		    &nvroot) == 0);
5350 	}
5351 
5352 	if (cbp->cb_verbose)
5353 		cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
5354 		    cbp->cb_name_flags);
5355 
5356 	print_pool(zhp, cbp);
5357 
5358 	if (cbp->cb_verbose)
5359 		print_list_stats(zhp, NULL, nvroot, cbp, 0);
5360 
5361 	return (0);
5362 }
5363 
5364 /*
5365  * Set the minimum pool/vdev name column width.  The width must be at least 9,
5366  * but may be as large as needed.
5367  */
5368 static int
5369 get_namewidth_list(zpool_handle_t *zhp, void *data)
5370 {
5371 	list_cbdata_t *cb = data;
5372 	int width;
5373 
5374 	width = get_namewidth(zhp, cb->cb_namewidth, cb->cb_name_flags,
5375 	    cb->cb_verbose);
5376 
5377 	if (width < 9)
5378 		width = 9;
5379 
5380 	cb->cb_namewidth = width;
5381 
5382 	return (0);
5383 }
5384 
5385 /*
5386  * zpool list [-gHLP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
5387  *
5388  *	-g	Display guid for individual vdev name.
5389  *	-H	Scripted mode.  Don't display headers, and separate properties
5390  *		by a single tab.
5391  *	-L	Follow links when resolving vdev path name.
5392  *	-o	List of properties to display.  Defaults to
5393  *		"name,size,allocated,free,expandsize,fragmentation,capacity,"
5394  *		"dedupratio,health,altroot"
5395  *	-p	Diplay values in parsable (exact) format.
5396  *	-P	Display full path for vdev name.
5397  *	-T	Display a timestamp in date(1) or Unix format
5398  *
5399  * List all pools in the system, whether or not they're healthy.  Output space
5400  * statistics for each one, as well as health status summary.
5401  */
5402 int
5403 zpool_do_list(int argc, char **argv)
5404 {
5405 	int c;
5406 	int ret;
5407 	list_cbdata_t cb = { 0 };
5408 	static char default_props[] =
5409 	    "name,size,allocated,free,checkpoint,expandsize,fragmentation,"
5410 	    "capacity,dedupratio,health,altroot";
5411 	char *props = default_props;
5412 	float interval = 0;
5413 	unsigned long count = 0;
5414 	zpool_list_t *list;
5415 	boolean_t first = B_TRUE;
5416 
5417 	/* check options */
5418 	while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
5419 		switch (c) {
5420 		case 'g':
5421 			cb.cb_name_flags |= VDEV_NAME_GUID;
5422 			break;
5423 		case 'H':
5424 			cb.cb_scripted = B_TRUE;
5425 			break;
5426 		case 'L':
5427 			cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
5428 			break;
5429 		case 'o':
5430 			props = optarg;
5431 			break;
5432 		case 'P':
5433 			cb.cb_name_flags |= VDEV_NAME_PATH;
5434 			break;
5435 		case 'p':
5436 			cb.cb_literal = B_TRUE;
5437 			break;
5438 		case 'T':
5439 			get_timestamp_arg(*optarg);
5440 			break;
5441 		case 'v':
5442 			cb.cb_verbose = B_TRUE;
5443 			cb.cb_namewidth = 8;	/* 8 until precalc is avail */
5444 			break;
5445 		case ':':
5446 			(void) fprintf(stderr, gettext("missing argument for "
5447 			    "'%c' option\n"), optopt);
5448 			usage(B_FALSE);
5449 			break;
5450 		case '?':
5451 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5452 			    optopt);
5453 			usage(B_FALSE);
5454 		}
5455 	}
5456 
5457 	argc -= optind;
5458 	argv += optind;
5459 
5460 	get_interval_count(&argc, argv, &interval, &count);
5461 
5462 	if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
5463 		usage(B_FALSE);
5464 
5465 	for (;;) {
5466 		if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
5467 		    &ret)) == NULL)
5468 			return (1);
5469 
5470 		if (pool_list_count(list) == 0)
5471 			break;
5472 
5473 		cb.cb_namewidth = 0;
5474 		(void) pool_list_iter(list, B_FALSE, get_namewidth_list, &cb);
5475 
5476 		if (timestamp_fmt != NODATE)
5477 			print_timestamp(timestamp_fmt);
5478 
5479 		if (!cb.cb_scripted && (first || cb.cb_verbose)) {
5480 			print_header(&cb);
5481 			first = B_FALSE;
5482 		}
5483 		ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
5484 
5485 		if (interval == 0)
5486 			break;
5487 
5488 		if (count != 0 && --count == 0)
5489 			break;
5490 
5491 		pool_list_free(list);
5492 		(void) fsleep(interval);
5493 	}
5494 
5495 	if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
5496 		(void) printf(gettext("no pools available\n"));
5497 		ret = 0;
5498 	}
5499 
5500 	pool_list_free(list);
5501 	zprop_free_list(cb.cb_proplist);
5502 	return (ret);
5503 }
5504 
5505 static int
5506 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
5507 {
5508 	boolean_t force = B_FALSE;
5509 	int c;
5510 	nvlist_t *nvroot;
5511 	char *poolname, *old_disk, *new_disk;
5512 	zpool_handle_t *zhp;
5513 	zpool_boot_label_t boot_type;
5514 	uint64_t boot_size;
5515 	nvlist_t *props = NULL;
5516 	char *propval;
5517 	int ret;
5518 
5519 	/* check options */
5520 	while ((c = getopt(argc, argv, "fo:")) != -1) {
5521 		switch (c) {
5522 		case 'f':
5523 			force = B_TRUE;
5524 			break;
5525 		case 'o':
5526 			if ((propval = strchr(optarg, '=')) == NULL) {
5527 				(void) fprintf(stderr, gettext("missing "
5528 				    "'=' for -o option\n"));
5529 				usage(B_FALSE);
5530 			}
5531 			*propval = '\0';
5532 			propval++;
5533 
5534 			if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
5535 			    (add_prop_list(optarg, propval, &props, B_TRUE)))
5536 				usage(B_FALSE);
5537 			break;
5538 		case '?':
5539 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5540 			    optopt);
5541 			usage(B_FALSE);
5542 		}
5543 	}
5544 
5545 	argc -= optind;
5546 	argv += optind;
5547 
5548 	/* get pool name and check number of arguments */
5549 	if (argc < 1) {
5550 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
5551 		usage(B_FALSE);
5552 	}
5553 
5554 	poolname = argv[0];
5555 
5556 	if (argc < 2) {
5557 		(void) fprintf(stderr,
5558 		    gettext("missing <device> specification\n"));
5559 		usage(B_FALSE);
5560 	}
5561 
5562 	old_disk = argv[1];
5563 
5564 	if (argc < 3) {
5565 		if (!replacing) {
5566 			(void) fprintf(stderr,
5567 			    gettext("missing <new_device> specification\n"));
5568 			usage(B_FALSE);
5569 		}
5570 		new_disk = old_disk;
5571 		argc -= 1;
5572 		argv += 1;
5573 	} else {
5574 		new_disk = argv[2];
5575 		argc -= 2;
5576 		argv += 2;
5577 	}
5578 
5579 	if (argc > 1) {
5580 		(void) fprintf(stderr, gettext("too many arguments\n"));
5581 		usage(B_FALSE);
5582 	}
5583 
5584 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5585 		return (1);
5586 
5587 	if (zpool_get_config(zhp, NULL) == NULL) {
5588 		(void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
5589 		    poolname);
5590 		zpool_close(zhp);
5591 		return (1);
5592 	}
5593 
5594 	if (zpool_is_bootable(zhp))
5595 		boot_type = ZPOOL_COPY_BOOT_LABEL;
5596 	else
5597 		boot_type = ZPOOL_NO_BOOT_LABEL;
5598 
5599 	boot_size = zpool_get_prop_int(zhp, ZPOOL_PROP_BOOTSIZE, NULL);
5600 
5601 	/* unless manually specified use "ashift" pool property (if set) */
5602 	if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
5603 		int intval;
5604 		zprop_source_t src;
5605 		char strval[ZPOOL_MAXPROPLEN];
5606 
5607 		intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
5608 		if (src != ZPROP_SRC_DEFAULT) {
5609 			(void) sprintf(strval, "%" PRId32, intval);
5610 			verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
5611 			    &props, B_TRUE) == 0);
5612 		}
5613 	}
5614 
5615 	nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
5616 	    boot_type, boot_size, argc, argv);
5617 	if (nvroot == NULL) {
5618 		zpool_close(zhp);
5619 		return (1);
5620 	}
5621 
5622 	ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing);
5623 
5624 	nvlist_free(nvroot);
5625 	zpool_close(zhp);
5626 
5627 	return (ret);
5628 }
5629 
5630 /*
5631  * zpool replace [-f] <pool> <device> <new_device>
5632  *
5633  *	-f	Force attach, even if <new_device> appears to be in use.
5634  *
5635  * Replace <device> with <new_device>.
5636  */
5637 /* ARGSUSED */
5638 int
5639 zpool_do_replace(int argc, char **argv)
5640 {
5641 	return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
5642 }
5643 
5644 /*
5645  * zpool attach [-f] [-o property=value] <pool> <device> <new_device>
5646  *
5647  *	-f	Force attach, even if <new_device> appears to be in use.
5648  *	-o	Set property=value.
5649  *
5650  * Attach <new_device> to the mirror containing <device>.  If <device> is not
5651  * part of a mirror, then <device> will be transformed into a mirror of
5652  * <device> and <new_device>.  In either case, <new_device> will begin life
5653  * with a DTL of [0, now], and will immediately begin to resilver itself.
5654  */
5655 int
5656 zpool_do_attach(int argc, char **argv)
5657 {
5658 	return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
5659 }
5660 
5661 /*
5662  * zpool detach [-f] <pool> <device>
5663  *
5664  *	-f	Force detach of <device>, even if DTLs argue against it
5665  *		(not supported yet)
5666  *
5667  * Detach a device from a mirror.  The operation will be refused if <device>
5668  * is the last device in the mirror, or if the DTLs indicate that this device
5669  * has the only valid copy of some data.
5670  */
5671 /* ARGSUSED */
5672 int
5673 zpool_do_detach(int argc, char **argv)
5674 {
5675 	int c;
5676 	char *poolname, *path;
5677 	zpool_handle_t *zhp;
5678 	int ret;
5679 
5680 	/* check options */
5681 	while ((c = getopt(argc, argv, "f")) != -1) {
5682 		switch (c) {
5683 		case 'f':
5684 		case '?':
5685 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5686 			    optopt);
5687 			usage(B_FALSE);
5688 		}
5689 	}
5690 
5691 	argc -= optind;
5692 	argv += optind;
5693 
5694 	/* get pool name and check number of arguments */
5695 	if (argc < 1) {
5696 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
5697 		usage(B_FALSE);
5698 	}
5699 
5700 	if (argc < 2) {
5701 		(void) fprintf(stderr,
5702 		    gettext("missing <device> specification\n"));
5703 		usage(B_FALSE);
5704 	}
5705 
5706 	poolname = argv[0];
5707 	path = argv[1];
5708 
5709 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5710 		return (1);
5711 
5712 	ret = zpool_vdev_detach(zhp, path);
5713 
5714 	zpool_close(zhp);
5715 
5716 	return (ret);
5717 }
5718 
5719 /*
5720  * zpool split [-gLnP] [-o prop=val] ...
5721  *		[-o mntopt] ...
5722  *		[-R altroot] <pool> <newpool> [<device> ...]
5723  *
5724  *	-g      Display guid for individual vdev name.
5725  *	-L	Follow links when resolving vdev path name.
5726  *	-n	Do not split the pool, but display the resulting layout if
5727  *		it were to be split.
5728  *	-o	Set property=value, or set mount options.
5729  *	-P	Display full path for vdev name.
5730  *	-R	Mount the split-off pool under an alternate root.
5731  *	-l	Load encryption keys while importing.
5732  *
5733  * Splits the named pool and gives it the new pool name.  Devices to be split
5734  * off may be listed, provided that no more than one device is specified
5735  * per top-level vdev mirror.  The newly split pool is left in an exported
5736  * state unless -R is specified.
5737  *
5738  * Restrictions: the top-level of the pool pool must only be made up of
5739  * mirrors; all devices in the pool must be healthy; no device may be
5740  * undergoing a resilvering operation.
5741  */
5742 int
5743 zpool_do_split(int argc, char **argv)
5744 {
5745 	char *srcpool, *newpool, *propval;
5746 	char *mntopts = NULL;
5747 	splitflags_t flags;
5748 	int c, ret = 0;
5749 	boolean_t loadkeys = B_FALSE;
5750 	zpool_handle_t *zhp;
5751 	nvlist_t *config, *props = NULL;
5752 
5753 	flags.dryrun = B_FALSE;
5754 	flags.import = B_FALSE;
5755 	flags.name_flags = 0;
5756 
5757 	/* check options */
5758 	while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
5759 		switch (c) {
5760 		case 'g':
5761 			flags.name_flags |= VDEV_NAME_GUID;
5762 			break;
5763 		case 'L':
5764 			flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
5765 			break;
5766 		case 'R':
5767 			flags.import = B_TRUE;
5768 			if (add_prop_list(
5769 			    zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
5770 			    &props, B_TRUE) != 0) {
5771 				nvlist_free(props);
5772 				usage(B_FALSE);
5773 			}
5774 			break;
5775 		case 'l':
5776 			loadkeys = B_TRUE;
5777 			break;
5778 		case 'n':
5779 			flags.dryrun = B_TRUE;
5780 			break;
5781 		case 'o':
5782 			if ((propval = strchr(optarg, '=')) != NULL) {
5783 				*propval = '\0';
5784 				propval++;
5785 				if (add_prop_list(optarg, propval,
5786 				    &props, B_TRUE) != 0) {
5787 					nvlist_free(props);
5788 					usage(B_FALSE);
5789 				}
5790 			} else {
5791 				mntopts = optarg;
5792 			}
5793 			break;
5794 		case 'P':
5795 			flags.name_flags |= VDEV_NAME_PATH;
5796 			break;
5797 		case ':':
5798 			(void) fprintf(stderr, gettext("missing argument for "
5799 			    "'%c' option\n"), optopt);
5800 			usage(B_FALSE);
5801 			break;
5802 		case '?':
5803 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5804 			    optopt);
5805 			usage(B_FALSE);
5806 			break;
5807 		}
5808 	}
5809 
5810 	if (!flags.import && mntopts != NULL) {
5811 		(void) fprintf(stderr, gettext("setting mntopts is only "
5812 		    "valid when importing the pool\n"));
5813 		usage(B_FALSE);
5814 	}
5815 
5816 	if (!flags.import && loadkeys) {
5817 		(void) fprintf(stderr, gettext("loading keys is only "
5818 		    "valid when importing the pool\n"));
5819 		usage(B_FALSE);
5820 	}
5821 
5822 	argc -= optind;
5823 	argv += optind;
5824 
5825 	if (argc < 1) {
5826 		(void) fprintf(stderr, gettext("Missing pool name\n"));
5827 		usage(B_FALSE);
5828 	}
5829 	if (argc < 2) {
5830 		(void) fprintf(stderr, gettext("Missing new pool name\n"));
5831 		usage(B_FALSE);
5832 	}
5833 
5834 	srcpool = argv[0];
5835 	newpool = argv[1];
5836 
5837 	argc -= 2;
5838 	argv += 2;
5839 
5840 	if ((zhp = zpool_open(g_zfs, srcpool)) == NULL)
5841 		return (1);
5842 
5843 	config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
5844 	if (config == NULL) {
5845 		ret = 1;
5846 	} else {
5847 		if (flags.dryrun) {
5848 			(void) printf(gettext("would create '%s' with the "
5849 			    "following layout:\n\n"), newpool);
5850 			print_vdev_tree(NULL, newpool, config, 0, "",
5851 			    flags.name_flags);
5852 		}
5853 		nvlist_free(config);
5854 	}
5855 
5856 	zpool_close(zhp);
5857 
5858 	if (ret != 0 || flags.dryrun || !flags.import)
5859 		return (ret);
5860 
5861 	/*
5862 	 * The split was successful. Now we need to open the new
5863 	 * pool and import it.
5864 	 */
5865 	if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL)
5866 		return (1);
5867 
5868 	if (loadkeys) {
5869 		ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
5870 		if (ret != 0)
5871 			ret = 1;
5872 	}
5873 
5874 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
5875 	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
5876 		ret = 1;
5877 		(void) fprintf(stderr, gettext("Split was successful, but "
5878 		    "the datasets could not all be mounted\n"));
5879 		(void) fprintf(stderr, gettext("Try doing '%s' with a "
5880 		    "different altroot\n"), "zpool import");
5881 	}
5882 	zpool_close(zhp);
5883 
5884 	return (ret);
5885 }
5886 
5887 
5888 
5889 /*
5890  * zpool online <pool> <device> ...
5891  */
5892 int
5893 zpool_do_online(int argc, char **argv)
5894 {
5895 	int c, i;
5896 	char *poolname;
5897 	zpool_handle_t *zhp;
5898 	int ret = 0;
5899 	vdev_state_t newstate;
5900 	int flags = 0;
5901 
5902 	/* check options */
5903 	while ((c = getopt(argc, argv, "et")) != -1) {
5904 		switch (c) {
5905 		case 'e':
5906 			flags |= ZFS_ONLINE_EXPAND;
5907 			break;
5908 		case 't':
5909 		case '?':
5910 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5911 			    optopt);
5912 			usage(B_FALSE);
5913 		}
5914 	}
5915 
5916 	argc -= optind;
5917 	argv += optind;
5918 
5919 	/* get pool name and check number of arguments */
5920 	if (argc < 1) {
5921 		(void) fprintf(stderr, gettext("missing pool name\n"));
5922 		usage(B_FALSE);
5923 	}
5924 	if (argc < 2) {
5925 		(void) fprintf(stderr, gettext("missing device name\n"));
5926 		usage(B_FALSE);
5927 	}
5928 
5929 	poolname = argv[0];
5930 
5931 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5932 		return (1);
5933 
5934 	for (i = 1; i < argc; i++) {
5935 		if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
5936 			if (newstate != VDEV_STATE_HEALTHY) {
5937 				(void) printf(gettext("warning: device '%s' "
5938 				    "onlined, but remains in faulted state\n"),
5939 				    argv[i]);
5940 				if (newstate == VDEV_STATE_FAULTED)
5941 					(void) printf(gettext("use 'zpool "
5942 					    "clear' to restore a faulted "
5943 					    "device\n"));
5944 				else
5945 					(void) printf(gettext("use 'zpool "
5946 					    "replace' to replace devices "
5947 					    "that are no longer present\n"));
5948 			}
5949 		} else {
5950 			ret = 1;
5951 		}
5952 	}
5953 
5954 	zpool_close(zhp);
5955 
5956 	return (ret);
5957 }
5958 
5959 /*
5960  * zpool offline [-ft] <pool> <device> ...
5961  *
5962  *	-f	Force the device into the offline state, even if doing
5963  *		so would appear to compromise pool availability.
5964  *		(not supported yet)
5965  *
5966  *	-t	Only take the device off-line temporarily.  The offline
5967  *		state will not be persistent across reboots.
5968  */
5969 /* ARGSUSED */
5970 int
5971 zpool_do_offline(int argc, char **argv)
5972 {
5973 	int c, i;
5974 	char *poolname;
5975 	zpool_handle_t *zhp;
5976 	int ret = 0;
5977 	boolean_t istmp = B_FALSE;
5978 
5979 	/* check options */
5980 	while ((c = getopt(argc, argv, "ft")) != -1) {
5981 		switch (c) {
5982 		case 't':
5983 			istmp = B_TRUE;
5984 			break;
5985 		case 'f':
5986 		case '?':
5987 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
5988 			    optopt);
5989 			usage(B_FALSE);
5990 		}
5991 	}
5992 
5993 	argc -= optind;
5994 	argv += optind;
5995 
5996 	/* get pool name and check number of arguments */
5997 	if (argc < 1) {
5998 		(void) fprintf(stderr, gettext("missing pool name\n"));
5999 		usage(B_FALSE);
6000 	}
6001 	if (argc < 2) {
6002 		(void) fprintf(stderr, gettext("missing device name\n"));
6003 		usage(B_FALSE);
6004 	}
6005 
6006 	poolname = argv[0];
6007 
6008 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6009 		return (1);
6010 
6011 	for (i = 1; i < argc; i++) {
6012 		if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
6013 			ret = 1;
6014 	}
6015 
6016 	zpool_close(zhp);
6017 
6018 	return (ret);
6019 }
6020 
6021 /*
6022  * zpool clear <pool> [device]
6023  *
6024  * Clear all errors associated with a pool or a particular device.
6025  */
6026 int
6027 zpool_do_clear(int argc, char **argv)
6028 {
6029 	int c;
6030 	int ret = 0;
6031 	boolean_t dryrun = B_FALSE;
6032 	boolean_t do_rewind = B_FALSE;
6033 	boolean_t xtreme_rewind = B_FALSE;
6034 	uint32_t rewind_policy = ZPOOL_NO_REWIND;
6035 	nvlist_t *policy = NULL;
6036 	zpool_handle_t *zhp;
6037 	char *pool, *device;
6038 
6039 	/* check options */
6040 	while ((c = getopt(argc, argv, "FnX")) != -1) {
6041 		switch (c) {
6042 		case 'F':
6043 			do_rewind = B_TRUE;
6044 			break;
6045 		case 'n':
6046 			dryrun = B_TRUE;
6047 			break;
6048 		case 'X':
6049 			xtreme_rewind = B_TRUE;
6050 			break;
6051 		case '?':
6052 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6053 			    optopt);
6054 			usage(B_FALSE);
6055 		}
6056 	}
6057 
6058 	argc -= optind;
6059 	argv += optind;
6060 
6061 	if (argc < 1) {
6062 		(void) fprintf(stderr, gettext("missing pool name\n"));
6063 		usage(B_FALSE);
6064 	}
6065 
6066 	if (argc > 2) {
6067 		(void) fprintf(stderr, gettext("too many arguments\n"));
6068 		usage(B_FALSE);
6069 	}
6070 
6071 	if ((dryrun || xtreme_rewind) && !do_rewind) {
6072 		(void) fprintf(stderr,
6073 		    gettext("-n or -X only meaningful with -F\n"));
6074 		usage(B_FALSE);
6075 	}
6076 	if (dryrun)
6077 		rewind_policy = ZPOOL_TRY_REWIND;
6078 	else if (do_rewind)
6079 		rewind_policy = ZPOOL_DO_REWIND;
6080 	if (xtreme_rewind)
6081 		rewind_policy |= ZPOOL_EXTREME_REWIND;
6082 
6083 	/* In future, further rewind policy choices can be passed along here */
6084 	if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
6085 	    nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
6086 	    rewind_policy) != 0) {
6087 		return (1);
6088 	}
6089 
6090 	pool = argv[0];
6091 	device = argc == 2 ? argv[1] : NULL;
6092 
6093 	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
6094 		nvlist_free(policy);
6095 		return (1);
6096 	}
6097 
6098 	if (zpool_clear(zhp, device, policy) != 0)
6099 		ret = 1;
6100 
6101 	zpool_close(zhp);
6102 
6103 	nvlist_free(policy);
6104 
6105 	return (ret);
6106 }
6107 
6108 /*
6109  * zpool reguid <pool>
6110  */
6111 int
6112 zpool_do_reguid(int argc, char **argv)
6113 {
6114 	int c;
6115 	char *poolname;
6116 	zpool_handle_t *zhp;
6117 	int ret = 0;
6118 
6119 	/* check options */
6120 	while ((c = getopt(argc, argv, "")) != -1) {
6121 		switch (c) {
6122 		case '?':
6123 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6124 			    optopt);
6125 			usage(B_FALSE);
6126 		}
6127 	}
6128 
6129 	argc -= optind;
6130 	argv += optind;
6131 
6132 	/* get pool name and check number of arguments */
6133 	if (argc < 1) {
6134 		(void) fprintf(stderr, gettext("missing pool name\n"));
6135 		usage(B_FALSE);
6136 	}
6137 
6138 	if (argc > 1) {
6139 		(void) fprintf(stderr, gettext("too many arguments\n"));
6140 		usage(B_FALSE);
6141 	}
6142 
6143 	poolname = argv[0];
6144 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6145 		return (1);
6146 
6147 	ret = zpool_reguid(zhp);
6148 
6149 	zpool_close(zhp);
6150 	return (ret);
6151 }
6152 
6153 
6154 /*
6155  * zpool reopen <pool>
6156  *
6157  * Reopen the pool so that the kernel can update the sizes of all vdevs.
6158  */
6159 int
6160 zpool_do_reopen(int argc, char **argv)
6161 {
6162 	int c;
6163 	int ret = 0;
6164 	zpool_handle_t *zhp;
6165 	char *pool;
6166 
6167 	/* check options */
6168 	while ((c = getopt(argc, argv, "")) != -1) {
6169 		switch (c) {
6170 		case '?':
6171 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6172 			    optopt);
6173 			usage(B_FALSE);
6174 		}
6175 	}
6176 
6177 	argc--;
6178 	argv++;
6179 
6180 	if (argc < 1) {
6181 		(void) fprintf(stderr, gettext("missing pool name\n"));
6182 		usage(B_FALSE);
6183 	}
6184 
6185 	if (argc > 1) {
6186 		(void) fprintf(stderr, gettext("too many arguments\n"));
6187 		usage(B_FALSE);
6188 	}
6189 
6190 	pool = argv[0];
6191 	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL)
6192 		return (1);
6193 
6194 	ret = zpool_reopen(zhp);
6195 	zpool_close(zhp);
6196 	return (ret);
6197 }
6198 
6199 typedef struct scrub_cbdata {
6200 	int	cb_type;
6201 	int	cb_argc;
6202 	char	**cb_argv;
6203 	pool_scrub_cmd_t cb_scrub_cmd;
6204 } scrub_cbdata_t;
6205 
6206 static boolean_t
6207 zpool_has_checkpoint(zpool_handle_t *zhp)
6208 {
6209 	nvlist_t *config, *nvroot;
6210 
6211 	config = zpool_get_config(zhp, NULL);
6212 
6213 	if (config != NULL) {
6214 		pool_checkpoint_stat_t *pcs = NULL;
6215 		uint_t c;
6216 
6217 		nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
6218 		(void) nvlist_lookup_uint64_array(nvroot,
6219 		    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
6220 
6221 		if (pcs == NULL || pcs->pcs_state == CS_NONE)
6222 			return (B_FALSE);
6223 
6224 		assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS ||
6225 		    pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
6226 		return (B_TRUE);
6227 	}
6228 
6229 	return (B_FALSE);
6230 }
6231 
6232 int
6233 scrub_callback(zpool_handle_t *zhp, void *data)
6234 {
6235 	scrub_cbdata_t *cb = data;
6236 	int err;
6237 
6238 	/*
6239 	 * Ignore faulted pools.
6240 	 */
6241 	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
6242 		(void) fprintf(stderr, gettext("cannot scan '%s': pool is "
6243 		    "currently unavailable\n"), zpool_get_name(zhp));
6244 		return (1);
6245 	}
6246 
6247 	err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
6248 
6249 	if (err == 0 && zpool_has_checkpoint(zhp) &&
6250 	    cb->cb_type == POOL_SCAN_SCRUB) {
6251 		(void) printf(gettext("warning: will not scrub state that "
6252 		    "belongs to the checkpoint of pool '%s'\n"),
6253 		    zpool_get_name(zhp));
6254 	}
6255 
6256 	return (err != 0);
6257 }
6258 
6259 /*
6260  * zpool scrub [-s | -p] <pool> ...
6261  *
6262  *	-s	Stop.  Stops any in-progress scrub.
6263  *	-p	Pause. Pause in-progress scrub.
6264  */
6265 int
6266 zpool_do_scrub(int argc, char **argv)
6267 {
6268 	int c;
6269 	scrub_cbdata_t cb;
6270 
6271 	cb.cb_type = POOL_SCAN_SCRUB;
6272 	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
6273 
6274 	/* check options */
6275 	while ((c = getopt(argc, argv, "sp")) != -1) {
6276 		switch (c) {
6277 		case 's':
6278 			cb.cb_type = POOL_SCAN_NONE;
6279 			break;
6280 		case 'p':
6281 			cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
6282 			break;
6283 		case '?':
6284 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6285 			    optopt);
6286 			usage(B_FALSE);
6287 		}
6288 	}
6289 
6290 	if (cb.cb_type == POOL_SCAN_NONE &&
6291 	    cb.cb_scrub_cmd == POOL_SCRUB_PAUSE) {
6292 		(void) fprintf(stderr, gettext("invalid option combination: "
6293 		    "-s and -p are mutually exclusive\n"));
6294 		usage(B_FALSE);
6295 	}
6296 
6297 	cb.cb_argc = argc;
6298 	cb.cb_argv = argv;
6299 	argc -= optind;
6300 	argv += optind;
6301 
6302 	if (argc < 1) {
6303 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
6304 		usage(B_FALSE);
6305 	}
6306 
6307 	return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
6308 }
6309 
6310 /*
6311  * zpool resilver <pool> ...
6312  *
6313  *	Restarts any in-progress resilver
6314  */
6315 int
6316 zpool_do_resilver(int argc, char **argv)
6317 {
6318 	int c;
6319 	scrub_cbdata_t cb;
6320 
6321 	cb.cb_type = POOL_SCAN_RESILVER;
6322 	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
6323 	cb.cb_argc = argc;
6324 	cb.cb_argv = argv;
6325 
6326 	/* check options */
6327 	while ((c = getopt(argc, argv, "")) != -1) {
6328 		switch (c) {
6329 		case '?':
6330 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
6331 			    optopt);
6332 			usage(B_FALSE);
6333 		}
6334 	}
6335 
6336 	argc -= optind;
6337 	argv += optind;
6338 
6339 	if (argc < 1) {
6340 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
6341 		usage(B_FALSE);
6342 	}
6343 
6344 	return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
6345 }
6346 
6347 /*
6348  * zpool trim [-d] [-r <rate>] [-c | -s] <pool> [<device> ...]
6349  *
6350  *	-c		Cancel. Ends any in-progress trim.
6351  *	-d		Secure trim.  Requires kernel and device support.
6352  *	-r <rate>	Sets the TRIM rate in bytes (per second). Supports
6353  *			adding a multiplier suffix such as 'k' or 'm'.
6354  *	-s		Suspend. TRIM can then be restarted with no flags.
6355  */
6356 int
6357 zpool_do_trim(int argc, char **argv)
6358 {
6359 	struct option long_options[] = {
6360 		{"cancel",	no_argument,		NULL,	'c'},
6361 		{"secure",	no_argument,		NULL,	'd'},
6362 		{"rate",	required_argument,	NULL,	'r'},
6363 		{"suspend",	no_argument,		NULL,	's'},
6364 		{0, 0, 0, 0}
6365 	};
6366 
6367 	pool_trim_func_t cmd_type = POOL_TRIM_START;
6368 	uint64_t rate = 0;
6369 	boolean_t secure = B_FALSE;
6370 
6371 	int c;
6372 	while ((c = getopt_long(argc, argv, "cdr:s", long_options, NULL))
6373 	    != -1) {
6374 		switch (c) {
6375 		case 'c':
6376 			if (cmd_type != POOL_TRIM_START &&
6377 			    cmd_type != POOL_TRIM_CANCEL) {
6378 				(void) fprintf(stderr, gettext("-c cannot be "
6379 				    "combined with other options\n"));
6380 				usage(B_FALSE);
6381 			}
6382 			cmd_type = POOL_TRIM_CANCEL;
6383 			break;
6384 		case 'd':
6385 			if (cmd_type != POOL_TRIM_START) {
6386 				(void) fprintf(stderr, gettext("-d cannot be "
6387 				    "combined with the -c or -s options\n"));
6388 				usage(B_FALSE);
6389 			}
6390 			secure = B_TRUE;
6391 			break;
6392 		case 'r':
6393 			if (cmd_type != POOL_TRIM_START) {
6394 				(void) fprintf(stderr, gettext("-r cannot be "
6395 				    "combined with the -c or -s options\n"));
6396 				usage(B_FALSE);
6397 			}
6398 			if (zfs_nicestrtonum(NULL, optarg, &rate) == -1) {
6399 				(void) fprintf(stderr,
6400 				    gettext("invalid value for rate\n"));
6401 				usage(B_FALSE);
6402 			}
6403 			break;
6404 		case 's':
6405 			if (cmd_type != POOL_TRIM_START &&
6406 			    cmd_type != POOL_TRIM_SUSPEND) {
6407 				(void) fprintf(stderr, gettext("-s cannot be "
6408 				    "combined with other options\n"));
6409 				usage(B_FALSE);
6410 			}
6411 			cmd_type = POOL_TRIM_SUSPEND;
6412 			break;
6413 		case '?':
6414 			if (optopt != 0) {
6415 				(void) fprintf(stderr,
6416 				    gettext("invalid option '%c'\n"), optopt);
6417 			} else {
6418 				(void) fprintf(stderr,
6419 				    gettext("invalid option '%s'\n"),
6420 				    argv[optind - 1]);
6421 			}
6422 			usage(B_FALSE);
6423 		}
6424 	}
6425 
6426 	argc -= optind;
6427 	argv += optind;
6428 
6429 	if (argc < 1) {
6430 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
6431 		usage(B_FALSE);
6432 		return (-1);
6433 	}
6434 
6435 	char *poolname = argv[0];
6436 	zpool_handle_t *zhp = zpool_open(g_zfs, poolname);
6437 	if (zhp == NULL)
6438 		return (-1);
6439 
6440 	trimflags_t trim_flags = {
6441 		.secure = secure,
6442 		.rate = rate,
6443 	};
6444 
6445 	nvlist_t *vdevs = fnvlist_alloc();
6446 	if (argc == 1) {
6447 		/* no individual leaf vdevs specified, so add them all */
6448 		nvlist_t *config = zpool_get_config(zhp, NULL);
6449 		nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
6450 		    ZPOOL_CONFIG_VDEV_TREE);
6451 		zpool_collect_leaves(zhp, nvroot, vdevs);
6452 		trim_flags.fullpool = B_TRUE;
6453 	} else {
6454 		trim_flags.fullpool = B_FALSE;
6455 		for (int i = 1; i < argc; i++) {
6456 			fnvlist_add_boolean(vdevs, argv[i]);
6457 		}
6458 	}
6459 
6460 	int error = zpool_trim(zhp, cmd_type, vdevs, &trim_flags);
6461 
6462 	fnvlist_free(vdevs);
6463 	zpool_close(zhp);
6464 
6465 	return (error);
6466 }
6467 
6468 /*
6469  * zpool initialize [-c | -s] <pool> [<vdev> ...]
6470  * Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
6471  * if none specified.
6472  *
6473  *	-c	Cancel. Ends active initializing.
6474  *	-s	Suspend. Initializing can then be restarted with no flags.
6475  */
6476 int
6477 zpool_do_initialize(int argc, char **argv)
6478 {
6479 	int c;
6480 	char *poolname;
6481 	zpool_handle_t *zhp;
6482 	nvlist_t *vdevs;
6483 	int err = 0;
6484 
6485 	struct option long_options[] = {
6486 		{"cancel",	no_argument,		NULL, 'c'},
6487 		{"suspend",	no_argument,		NULL, 's'},
6488 		{0, 0, 0, 0}
6489 	};
6490 
6491 	pool_initialize_func_t cmd_type = POOL_INITIALIZE_START;
6492 	while ((c = getopt_long(argc, argv, "cs", long_options, NULL)) != -1) {
6493 		switch (c) {
6494 		case 'c':
6495 			if (cmd_type != POOL_INITIALIZE_START &&
6496 			    cmd_type != POOL_INITIALIZE_CANCEL) {
6497 				(void) fprintf(stderr, gettext("-c cannot be "
6498 				    "combined with other options\n"));
6499 				usage(B_FALSE);
6500 			}
6501 			cmd_type = POOL_INITIALIZE_CANCEL;
6502 			break;
6503 		case 's':
6504 			if (cmd_type != POOL_INITIALIZE_START &&
6505 			    cmd_type != POOL_INITIALIZE_SUSPEND) {
6506 				(void) fprintf(stderr, gettext("-s cannot be "
6507 				    "combined with other options\n"));
6508 				usage(B_FALSE);
6509 			}
6510 			cmd_type = POOL_INITIALIZE_SUSPEND;
6511 			break;
6512 		case '?':
6513 			if (optopt != 0) {
6514 				(void) fprintf(stderr,
6515 				    gettext("invalid option '%c'\n"), optopt);
6516 			} else {
6517 				(void) fprintf(stderr,
6518 				    gettext("invalid option '%s'\n"),
6519 				    argv[optind - 1]);
6520 			}
6521 			usage(B_FALSE);
6522 		}
6523 	}
6524 
6525 	argc -= optind;
6526 	argv += optind;
6527 
6528 	if (argc < 1) {
6529 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
6530 		usage(B_FALSE);
6531 		return (-1);
6532 	}
6533 
6534 	poolname = argv[0];
6535 	zhp = zpool_open(g_zfs, poolname);
6536 	if (zhp == NULL)
6537 		return (-1);
6538 
6539 	vdevs = fnvlist_alloc();
6540 	if (argc == 1) {
6541 		/* no individual leaf vdevs specified, so add them all */
6542 		nvlist_t *config = zpool_get_config(zhp, NULL);
6543 		nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
6544 		    ZPOOL_CONFIG_VDEV_TREE);
6545 		zpool_collect_leaves(zhp, nvroot, vdevs);
6546 	} else {
6547 		for (int i = 1; i < argc; i++) {
6548 			fnvlist_add_boolean(vdevs, argv[i]);
6549 		}
6550 	}
6551 
6552 	err = zpool_initialize(zhp, cmd_type, vdevs);
6553 
6554 	fnvlist_free(vdevs);
6555 	zpool_close(zhp);
6556 
6557 	return (err);
6558 }
6559 
6560 /*
6561  * Print out detailed scrub status.
6562  */
6563 static void
6564 print_scan_status(pool_scan_stat_t *ps)
6565 {
6566 	time_t start, end, pause;
6567 	uint64_t total_secs_left;
6568 	uint64_t elapsed, secs_left, mins_left, hours_left, days_left;
6569 	uint64_t pass_scanned, scanned, pass_issued, issued, total;
6570 	uint_t scan_rate, issue_rate;
6571 	double fraction_done;
6572 	char processed_buf[7], scanned_buf[7], issued_buf[7], total_buf[7];
6573 	char srate_buf[7], irate_buf[7];
6574 
6575 	(void) printf(gettext("  scan: "));
6576 
6577 	/* If there's never been a scan, there's not much to say. */
6578 	if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
6579 	    ps->pss_func >= POOL_SCAN_FUNCS) {
6580 		(void) printf(gettext("none requested\n"));
6581 		return;
6582 	}
6583 
6584 	start = ps->pss_start_time;
6585 	end = ps->pss_end_time;
6586 	pause = ps->pss_pass_scrub_pause;
6587 
6588 	zfs_nicenum(ps->pss_processed, processed_buf, sizeof (processed_buf));
6589 
6590 	assert(ps->pss_func == POOL_SCAN_SCRUB ||
6591 	    ps->pss_func == POOL_SCAN_RESILVER);
6592 
6593 	/*
6594 	 * Scan is finished or canceled.
6595 	 */
6596 	if (ps->pss_state == DSS_FINISHED) {
6597 		total_secs_left = end - start;
6598 		days_left = total_secs_left / 60 / 60 / 24;
6599 		hours_left = (total_secs_left / 60 / 60) % 24;
6600 		mins_left = (total_secs_left / 60) % 60;
6601 		secs_left = (total_secs_left % 60);
6602 
6603 		if (ps->pss_func == POOL_SCAN_SCRUB) {
6604 			(void) printf(gettext("scrub repaired %s "
6605 			    "in %llu days %02llu:%02llu:%02llu "
6606 			    "with %llu errors on %s"), processed_buf,
6607 			    (u_longlong_t)days_left, (u_longlong_t)hours_left,
6608 			    (u_longlong_t)mins_left, (u_longlong_t)secs_left,
6609 			    (u_longlong_t)ps->pss_errors, ctime(&end));
6610 		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
6611 			(void) printf(gettext("resilvered %s "
6612 			    "in %llu days %02llu:%02llu:%02llu "
6613 			    "with %llu errors on %s"), processed_buf,
6614 			    (u_longlong_t)days_left, (u_longlong_t)hours_left,
6615 			    (u_longlong_t)mins_left, (u_longlong_t)secs_left,
6616 			    (u_longlong_t)ps->pss_errors, ctime(&end));
6617 		}
6618 		return;
6619 	} else if (ps->pss_state == DSS_CANCELED) {
6620 		if (ps->pss_func == POOL_SCAN_SCRUB) {
6621 			(void) printf(gettext("scrub canceled on %s"),
6622 			    ctime(&end));
6623 		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
6624 			(void) printf(gettext("resilver canceled on %s"),
6625 			    ctime(&end));
6626 		}
6627 		return;
6628 	}
6629 
6630 	assert(ps->pss_state == DSS_SCANNING);
6631 
6632 	/* Scan is in progress. Resilvers can't be paused. */
6633 	if (ps->pss_func == POOL_SCAN_SCRUB) {
6634 		if (pause == 0) {
6635 			(void) printf(gettext("scrub in progress since %s"),
6636 			    ctime(&start));
6637 		} else {
6638 			(void) printf(gettext("scrub paused since %s"),
6639 			    ctime(&pause));
6640 			(void) printf(gettext("\tscrub started on %s"),
6641 			    ctime(&start));
6642 		}
6643 	} else if (ps->pss_func == POOL_SCAN_RESILVER) {
6644 		(void) printf(gettext("resilver in progress since %s"),
6645 		    ctime(&start));
6646 	}
6647 
6648 	scanned = ps->pss_examined;
6649 	pass_scanned = ps->pss_pass_exam;
6650 	issued = ps->pss_issued;
6651 	pass_issued = ps->pss_pass_issued;
6652 	total = ps->pss_to_examine;
6653 
6654 	/* we are only done with a block once we have issued the IO for it  */
6655 	fraction_done = (double)issued / total;
6656 
6657 	/* elapsed time for this pass, rounding up to 1 if it's 0 */
6658 	elapsed = time(NULL) - ps->pss_pass_start;
6659 	elapsed -= ps->pss_pass_scrub_spent_paused;
6660 	elapsed = (elapsed != 0) ? elapsed : 1;
6661 
6662 	scan_rate = pass_scanned / elapsed;
6663 	issue_rate = pass_issued / elapsed;
6664 	total_secs_left = (issue_rate != 0 && total >= issued) ?
6665 	    ((total - issued) / issue_rate) : UINT64_MAX;
6666 
6667 	days_left = total_secs_left / 60 / 60 / 24;
6668 	hours_left = (total_secs_left / 60 / 60) % 24;
6669 	mins_left = (total_secs_left / 60) % 60;
6670 	secs_left = (total_secs_left % 60);
6671 
6672 	/* format all of the numbers we will be reporting */
6673 	zfs_nicenum(scanned, scanned_buf, sizeof (scanned_buf));
6674 	zfs_nicenum(issued, issued_buf, sizeof (issued_buf));
6675 	zfs_nicenum(total, total_buf, sizeof (total_buf));
6676 	zfs_nicenum(scan_rate, srate_buf, sizeof (srate_buf));
6677 	zfs_nicenum(issue_rate, irate_buf, sizeof (irate_buf));
6678 
6679 	/* do not print estimated time if we have a paused scrub */
6680 	if (pause == 0) {
6681 		(void) printf(gettext("\t%s scanned at %s/s, "
6682 		    "%s issued at %s/s, %s total\n"),
6683 		    scanned_buf, srate_buf, issued_buf, irate_buf, total_buf);
6684 	} else {
6685 		(void) printf(gettext("\t%s scanned, %s issued, %s total\n"),
6686 		    scanned_buf, issued_buf, total_buf);
6687 	}
6688 
6689 	if (ps->pss_func == POOL_SCAN_RESILVER) {
6690 		(void) printf(gettext("\t%s resilvered, %.2f%% done"),
6691 		    processed_buf, 100 * fraction_done);
6692 	} else if (ps->pss_func == POOL_SCAN_SCRUB) {
6693 		(void) printf(gettext("\t%s repaired, %.2f%% done"),
6694 		    processed_buf, 100 * fraction_done);
6695 	}
6696 
6697 	if (pause == 0) {
6698 		if (total_secs_left != UINT64_MAX &&
6699 		    issue_rate >= 10 * 1024 * 1024) {
6700 			(void) printf(gettext(", %llu days "
6701 			    "%02llu:%02llu:%02llu to go\n"),
6702 			    (u_longlong_t)days_left, (u_longlong_t)hours_left,
6703 			    (u_longlong_t)mins_left, (u_longlong_t)secs_left);
6704 		} else {
6705 			(void) printf(gettext(", no estimated "
6706 			    "completion time\n"));
6707 		}
6708 	} else {
6709 		(void) printf(gettext("\n"));
6710 	}
6711 }
6712 
6713 /*
6714  * As we don't scrub checkpointed blocks, we want to warn the
6715  * user that we skipped scanning some blocks if a checkpoint exists
6716  * or existed at any time during the scan.
6717  */
6718 static void
6719 print_checkpoint_scan_warning(pool_scan_stat_t *ps, pool_checkpoint_stat_t *pcs)
6720 {
6721 	if (ps == NULL || pcs == NULL)
6722 		return;
6723 
6724 	if (pcs->pcs_state == CS_NONE ||
6725 	    pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
6726 		return;
6727 
6728 	assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);
6729 
6730 	if (ps->pss_state == DSS_NONE)
6731 		return;
6732 
6733 	if ((ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) &&
6734 	    ps->pss_end_time < pcs->pcs_start_time)
6735 		return;
6736 
6737 	if (ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) {
6738 		(void) printf(gettext("    scan warning: skipped blocks "
6739 		    "that are only referenced by the checkpoint.\n"));
6740 	} else {
6741 		assert(ps->pss_state == DSS_SCANNING);
6742 		(void) printf(gettext("    scan warning: skipping blocks "
6743 		    "that are only referenced by the checkpoint.\n"));
6744 	}
6745 }
6746 
6747 /*
6748  * Print out detailed removal status.
6749  */
6750 static void
6751 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
6752 {
6753 	char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
6754 	time_t start, end;
6755 	nvlist_t *config, *nvroot;
6756 	nvlist_t **child;
6757 	uint_t children;
6758 	char *vdev_name;
6759 
6760 	if (prs == NULL || prs->prs_state == DSS_NONE)
6761 		return;
6762 
6763 	/*
6764 	 * Determine name of vdev.
6765 	 */
6766 	config = zpool_get_config(zhp, NULL);
6767 	nvroot = fnvlist_lookup_nvlist(config,
6768 	    ZPOOL_CONFIG_VDEV_TREE);
6769 	verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
6770 	    &child, &children) == 0);
6771 	assert(prs->prs_removing_vdev < children);
6772 	vdev_name = zpool_vdev_name(g_zfs, zhp,
6773 	    child[prs->prs_removing_vdev], B_TRUE);
6774 
6775 	(void) printf(gettext("remove: "));
6776 
6777 	start = prs->prs_start_time;
6778 	end = prs->prs_end_time;
6779 	zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
6780 
6781 	/*
6782 	 * Removal is finished or canceled.
6783 	 */
6784 	if (prs->prs_state == DSS_FINISHED) {
6785 		uint64_t minutes_taken = (end - start) / 60;
6786 
6787 		(void) printf(gettext("Removal of vdev %llu copied %s "
6788 		    "in %lluh%um, completed on %s"),
6789 		    (longlong_t)prs->prs_removing_vdev,
6790 		    copied_buf,
6791 		    (u_longlong_t)(minutes_taken / 60),
6792 		    (uint_t)(minutes_taken % 60),
6793 		    ctime((time_t *)&end));
6794 	} else if (prs->prs_state == DSS_CANCELED) {
6795 		(void) printf(gettext("Removal of %s canceled on %s"),
6796 		    vdev_name, ctime(&end));
6797 	} else {
6798 		uint64_t copied, total, elapsed, mins_left, hours_left;
6799 		double fraction_done;
6800 		uint_t rate;
6801 
6802 		assert(prs->prs_state == DSS_SCANNING);
6803 
6804 		/*
6805 		 * Removal is in progress.
6806 		 */
6807 		(void) printf(gettext(
6808 		    "Evacuation of %s in progress since %s"),
6809 		    vdev_name, ctime(&start));
6810 
6811 		copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
6812 		total = prs->prs_to_copy;
6813 		fraction_done = (double)copied / total;
6814 
6815 		/* elapsed time for this pass */
6816 		elapsed = time(NULL) - prs->prs_start_time;
6817 		elapsed = elapsed > 0 ? elapsed : 1;
6818 		rate = copied / elapsed;
6819 		rate = rate > 0 ? rate : 1;
6820 		mins_left = ((total - copied) / rate) / 60;
6821 		hours_left = mins_left / 60;
6822 
6823 		zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
6824 		zfs_nicenum(total, total_buf, sizeof (total_buf));
6825 		zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
6826 
6827 		/*
6828 		 * do not print estimated time if hours_left is more than
6829 		 * 30 days
6830 		 */
6831 		(void) printf(gettext("    %s copied out of %s at %s/s, "
6832 		    "%.2f%% done"),
6833 		    examined_buf, total_buf, rate_buf, 100 * fraction_done);
6834 		if (hours_left < (30 * 24)) {
6835 			(void) printf(gettext(", %lluh%um to go\n"),
6836 			    (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
6837 		} else {
6838 			(void) printf(gettext(
6839 			    ", (copy is slow, no estimated time)\n"));
6840 		}
6841 	}
6842 
6843 	if (prs->prs_mapping_memory > 0) {
6844 		char mem_buf[7];
6845 		zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
6846 		(void) printf(gettext("    %s memory used for "
6847 		    "removed device mappings\n"),
6848 		    mem_buf);
6849 	}
6850 }
6851 
6852 static void
6853 print_checkpoint_status(pool_checkpoint_stat_t *pcs)
6854 {
6855 	time_t start;
6856 	char space_buf[7];
6857 
6858 	if (pcs == NULL || pcs->pcs_state == CS_NONE)
6859 		return;
6860 
6861 	(void) printf(gettext("checkpoint: "));
6862 
6863 	start = pcs->pcs_start_time;
6864 	zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));
6865 
6866 	if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
6867 		char *date = ctime(&start);
6868 
6869 		/*
6870 		 * ctime() adds a newline at the end of the generated
6871 		 * string, thus the weird format specifier and the
6872 		 * strlen() call used to chop it off from the output.
6873 		 */
6874 		(void) printf(gettext("created %.*s, consumes %s\n"),
6875 		    strlen(date) - 1, date, space_buf);
6876 		return;
6877 	}
6878 
6879 	assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
6880 
6881 	(void) printf(gettext("discarding, %s remaining.\n"),
6882 	    space_buf);
6883 }
6884 
6885 static void
6886 print_error_log(zpool_handle_t *zhp)
6887 {
6888 	nvlist_t *nverrlist = NULL;
6889 	nvpair_t *elem;
6890 	char *pathname;
6891 	size_t len = MAXPATHLEN * 2;
6892 
6893 	if (zpool_get_errlog(zhp, &nverrlist) != 0) {
6894 		(void) printf("errors: List of errors unavailable "
6895 		    "(insufficient privileges)\n");
6896 		return;
6897 	}
6898 
6899 	(void) printf("errors: Permanent errors have been "
6900 	    "detected in the following files:\n\n");
6901 
6902 	pathname = safe_malloc(len);
6903 	elem = NULL;
6904 	while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
6905 		nvlist_t *nv;
6906 		uint64_t dsobj, obj;
6907 
6908 		verify(nvpair_value_nvlist(elem, &nv) == 0);
6909 		verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
6910 		    &dsobj) == 0);
6911 		verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
6912 		    &obj) == 0);
6913 		zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
6914 		(void) printf("%7s %s\n", "", pathname);
6915 	}
6916 	free(pathname);
6917 	nvlist_free(nverrlist);
6918 }
6919 
6920 static void
6921 print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
6922     uint_t nspares)
6923 {
6924 	uint_t i;
6925 	char *name;
6926 
6927 	if (nspares == 0)
6928 		return;
6929 
6930 	(void) printf(gettext("\tspares\n"));
6931 
6932 	for (i = 0; i < nspares; i++) {
6933 		name = zpool_vdev_name(g_zfs, zhp, spares[i],
6934 		    cb->cb_name_flags);
6935 		print_status_config(zhp, cb, name, spares[i], 2, B_TRUE);
6936 		free(name);
6937 	}
6938 }
6939 
6940 static void
6941 print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
6942     uint_t nl2cache)
6943 {
6944 	uint_t i;
6945 	char *name;
6946 
6947 	if (nl2cache == 0)
6948 		return;
6949 
6950 	(void) printf(gettext("\tcache\n"));
6951 
6952 	for (i = 0; i < nl2cache; i++) {
6953 		name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
6954 		    cb->cb_name_flags);
6955 		print_status_config(zhp, cb, name, l2cache[i], 2, B_FALSE);
6956 		free(name);
6957 	}
6958 }
6959 
6960 static void
6961 print_dedup_stats(nvlist_t *config)
6962 {
6963 	ddt_histogram_t *ddh;
6964 	ddt_stat_t *dds;
6965 	ddt_object_t *ddo;
6966 	uint_t c;
6967 
6968 	/*
6969 	 * If the pool was faulted then we may not have been able to
6970 	 * obtain the config. Otherwise, if we have anything in the dedup
6971 	 * table continue processing the stats.
6972 	 */
6973 	if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
6974 	    (uint64_t **)&ddo, &c) != 0)
6975 		return;
6976 
6977 	(void) printf("\n");
6978 	(void) printf(gettext(" dedup: "));
6979 	if (ddo->ddo_count == 0) {
6980 		(void) printf(gettext("no DDT entries\n"));
6981 		return;
6982 	}
6983 
6984 	(void) printf("DDT entries %llu, size %llu on disk, %llu in core\n",
6985 	    (u_longlong_t)ddo->ddo_count,
6986 	    (u_longlong_t)ddo->ddo_dspace,
6987 	    (u_longlong_t)ddo->ddo_mspace);
6988 
6989 	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
6990 	    (uint64_t **)&dds, &c) == 0);
6991 	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
6992 	    (uint64_t **)&ddh, &c) == 0);
6993 	zpool_dump_ddt(dds, ddh);
6994 }
6995 
6996 /*
6997  * Display a summary of pool status.  Displays a summary such as:
6998  *
6999  *        pool: tank
7000  *	status: DEGRADED
7001  *	reason: One or more devices ...
7002  *         see: http://illumos.org/msg/ZFS-xxxx-01
7003  *	config:
7004  *		mirror		DEGRADED
7005  *                c1t0d0	OK
7006  *                c2t0d0	UNAVAIL
7007  *
7008  * When given the '-v' option, we print out the complete config.  If the '-e'
7009  * option is specified, then we print out error rate information as well.
7010  */
7011 int
7012 status_callback(zpool_handle_t *zhp, void *data)
7013 {
7014 	status_cbdata_t *cbp = data;
7015 	nvlist_t *config, *nvroot;
7016 	char *msgid;
7017 	int reason;
7018 	zpool_errata_t errata;
7019 	const char *health;
7020 	uint_t c;
7021 	vdev_stat_t *vs;
7022 
7023 	config = zpool_get_config(zhp, NULL);
7024 	reason = zpool_get_status(zhp, &msgid, &errata);
7025 
7026 	cbp->cb_count++;
7027 
7028 	/*
7029 	 * If we were given 'zpool status -x', only report those pools with
7030 	 * problems.
7031 	 */
7032 	if (cbp->cb_explain &&
7033 	    (reason == ZPOOL_STATUS_OK ||
7034 	    reason == ZPOOL_STATUS_VERSION_OLDER ||
7035 	    reason == ZPOOL_STATUS_FEAT_DISABLED)) {
7036 		if (!cbp->cb_allpools) {
7037 			(void) printf(gettext("pool '%s' is healthy\n"),
7038 			    zpool_get_name(zhp));
7039 			if (cbp->cb_first)
7040 				cbp->cb_first = B_FALSE;
7041 		}
7042 		return (0);
7043 	}
7044 
7045 	if (cbp->cb_first)
7046 		cbp->cb_first = B_FALSE;
7047 	else
7048 		(void) printf("\n");
7049 
7050 	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
7051 	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
7052 	    (uint64_t **)&vs, &c) == 0);
7053 	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
7054 
7055 	(void) printf(gettext("  pool: %s\n"), zpool_get_name(zhp));
7056 	(void) printf(gettext(" state: %s\n"), health);
7057 
7058 	switch (reason) {
7059 	case ZPOOL_STATUS_MISSING_DEV_R:
7060 		(void) printf(gettext("status: One or more devices could not "
7061 		    "be opened.  Sufficient replicas exist for\n\tthe pool to "
7062 		    "continue functioning in a degraded state.\n"));
7063 		(void) printf(gettext("action: Attach the missing device and "
7064 		    "online it using 'zpool online'.\n"));
7065 		break;
7066 
7067 	case ZPOOL_STATUS_MISSING_DEV_NR:
7068 		(void) printf(gettext("status: One or more devices could not "
7069 		    "be opened.  There are insufficient\n\treplicas for the "
7070 		    "pool to continue functioning.\n"));
7071 		(void) printf(gettext("action: Attach the missing device and "
7072 		    "online it using 'zpool online'.\n"));
7073 		break;
7074 
7075 	case ZPOOL_STATUS_CORRUPT_LABEL_R:
7076 		(void) printf(gettext("status: One or more devices could not "
7077 		    "be used because the label is missing or\n\tinvalid.  "
7078 		    "Sufficient replicas exist for the pool to continue\n\t"
7079 		    "functioning in a degraded state.\n"));
7080 		(void) printf(gettext("action: Replace the device using "
7081 		    "'zpool replace'.\n"));
7082 		break;
7083 
7084 	case ZPOOL_STATUS_CORRUPT_LABEL_NR:
7085 		(void) printf(gettext("status: One or more devices could not "
7086 		    "be used because the label is missing \n\tor invalid.  "
7087 		    "There are insufficient replicas for the pool to "
7088 		    "continue\n\tfunctioning.\n"));
7089 		zpool_explain_recover(zpool_get_handle(zhp),
7090 		    zpool_get_name(zhp), reason, config);
7091 		break;
7092 
7093 	case ZPOOL_STATUS_FAILING_DEV:
7094 		(void) printf(gettext("status: One or more devices has "
7095 		    "experienced an unrecoverable error.  An\n\tattempt was "
7096 		    "made to correct the error.  Applications are "
7097 		    "unaffected.\n"));
7098 		(void) printf(gettext("action: Determine if the device needs "
7099 		    "to be replaced, and clear the errors\n\tusing "
7100 		    "'zpool clear' or replace the device with 'zpool "
7101 		    "replace'.\n"));
7102 		break;
7103 
7104 	case ZPOOL_STATUS_OFFLINE_DEV:
7105 		(void) printf(gettext("status: One or more devices has "
7106 		    "been taken offline by the administrator.\n\tSufficient "
7107 		    "replicas exist for the pool to continue functioning in "
7108 		    "a\n\tdegraded state.\n"));
7109 		(void) printf(gettext("action: Online the device using "
7110 		    "'zpool online' or replace the device with\n\t'zpool "
7111 		    "replace'.\n"));
7112 		break;
7113 
7114 	case ZPOOL_STATUS_REMOVED_DEV:
7115 		(void) printf(gettext("status: One or more devices has "
7116 		    "been removed by the administrator.\n\tSufficient "
7117 		    "replicas exist for the pool to continue functioning in "
7118 		    "a\n\tdegraded state.\n"));
7119 		(void) printf(gettext("action: Online the device using "
7120 		    "'zpool online' or replace the device with\n\t'zpool "
7121 		    "replace'.\n"));
7122 		break;
7123 
7124 	case ZPOOL_STATUS_RESILVERING:
7125 		(void) printf(gettext("status: One or more devices is "
7126 		    "currently being resilvered.  The pool will\n\tcontinue "
7127 		    "to function, possibly in a degraded state.\n"));
7128 		(void) printf(gettext("action: Wait for the resilver to "
7129 		    "complete.\n"));
7130 		break;
7131 
7132 	case ZPOOL_STATUS_CORRUPT_DATA:
7133 		(void) printf(gettext("status: One or more devices has "
7134 		    "experienced an error resulting in data\n\tcorruption.  "
7135 		    "Applications may be affected.\n"));
7136 		(void) printf(gettext("action: Restore the file in question "
7137 		    "if possible.  Otherwise restore the\n\tentire pool from "
7138 		    "backup.\n"));
7139 		break;
7140 
7141 	case ZPOOL_STATUS_CORRUPT_POOL:
7142 		(void) printf(gettext("status: The pool metadata is corrupted "
7143 		    "and the pool cannot be opened.\n"));
7144 		zpool_explain_recover(zpool_get_handle(zhp),
7145 		    zpool_get_name(zhp), reason, config);
7146 		break;
7147 
7148 	case ZPOOL_STATUS_VERSION_OLDER:
7149 		(void) printf(gettext("status: The pool is formatted using a "
7150 		    "legacy on-disk format.  The pool can\n\tstill be used, "
7151 		    "but some features are unavailable.\n"));
7152 		(void) printf(gettext("action: Upgrade the pool using 'zpool "
7153 		    "upgrade'.  Once this is done, the\n\tpool will no longer "
7154 		    "be accessible on software that does not support feature\n"
7155 		    "\tflags.\n"));
7156 		break;
7157 
7158 	case ZPOOL_STATUS_VERSION_NEWER:
7159 		(void) printf(gettext("status: The pool has been upgraded to a "
7160 		    "newer, incompatible on-disk version.\n\tThe pool cannot "
7161 		    "be accessed on this system.\n"));
7162 		(void) printf(gettext("action: Access the pool from a system "
7163 		    "running more recent software, or\n\trestore the pool from "
7164 		    "backup.\n"));
7165 		break;
7166 
7167 	case ZPOOL_STATUS_FEAT_DISABLED:
7168 		(void) printf(gettext("status: Some supported features are not "
7169 		    "enabled on the pool. The pool can\n\tstill be used, but "
7170 		    "some features are unavailable.\n"));
7171 		(void) printf(gettext("action: Enable all features using "
7172 		    "'zpool upgrade'. Once this is done,\n\tthe pool may no "
7173 		    "longer be accessible by software that does not support\n\t"
7174 		    "the features. See zpool-features(5) for details.\n"));
7175 		break;
7176 
7177 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
7178 		(void) printf(gettext("status: The pool cannot be accessed on "
7179 		    "this system because it uses the\n\tfollowing feature(s) "
7180 		    "not supported on this system:\n"));
7181 		zpool_print_unsup_feat(config);
7182 		(void) printf("\n");
7183 		(void) printf(gettext("action: Access the pool from a system "
7184 		    "that supports the required feature(s),\n\tor restore the "
7185 		    "pool from backup.\n"));
7186 		break;
7187 
7188 	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
7189 		(void) printf(gettext("status: The pool can only be accessed "
7190 		    "in read-only mode on this system. It\n\tcannot be "
7191 		    "accessed in read-write mode because it uses the "
7192 		    "following\n\tfeature(s) not supported on this system:\n"));
7193 		zpool_print_unsup_feat(config);
7194 		(void) printf("\n");
7195 		(void) printf(gettext("action: The pool cannot be accessed in "
7196 		    "read-write mode. Import the pool with\n"
7197 		    "\t\"-o readonly=on\", access the pool from a system that "
7198 		    "supports the\n\trequired feature(s), or restore the "
7199 		    "pool from backup.\n"));
7200 		break;
7201 
7202 	case ZPOOL_STATUS_FAULTED_DEV_R:
7203 		(void) printf(gettext("status: One or more devices are "
7204 		    "faulted in response to persistent errors.\n\tSufficient "
7205 		    "replicas exist for the pool to continue functioning "
7206 		    "in a\n\tdegraded state.\n"));
7207 		(void) printf(gettext("action: Replace the faulted device, "
7208 		    "or use 'zpool clear' to mark the device\n\trepaired.\n"));
7209 		break;
7210 
7211 	case ZPOOL_STATUS_FAULTED_DEV_NR:
7212 		(void) printf(gettext("status: One or more devices are "
7213 		    "faulted in response to persistent errors.  There are "
7214 		    "insufficient replicas for the pool to\n\tcontinue "
7215 		    "functioning.\n"));
7216 		(void) printf(gettext("action: Destroy and re-create the pool "
7217 		    "from a backup source.  Manually marking the device\n"
7218 		    "\trepaired using 'zpool clear' may allow some data "
7219 		    "to be recovered.\n"));
7220 		break;
7221 
7222 	case ZPOOL_STATUS_IO_FAILURE_MMP:
7223 		(void) printf(gettext("status: The pool is suspended because "
7224 		    "multihost writes failed or were delayed;\n\tanother "
7225 		    "system could import the pool undetected.\n"));
7226 		(void) printf(gettext("action: Make sure the pool's devices "
7227 		    "are connected, then reboot your system and\n\timport the "
7228 		    "pool.\n"));
7229 		break;
7230 
7231 	case ZPOOL_STATUS_IO_FAILURE_WAIT:
7232 	case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
7233 		(void) printf(gettext("status: One or more devices are "
7234 		    "faulted in response to IO failures.\n"));
7235 		(void) printf(gettext("action: Make sure the affected devices "
7236 		    "are connected, then run 'zpool clear'.\n"));
7237 		break;
7238 
7239 	case ZPOOL_STATUS_BAD_LOG:
7240 		(void) printf(gettext("status: An intent log record "
7241 		    "could not be read.\n"
7242 		    "\tWaiting for adminstrator intervention to fix the "
7243 		    "faulted pool.\n"));
7244 		(void) printf(gettext("action: Either restore the affected "
7245 		    "device(s) and run 'zpool online',\n"
7246 		    "\tor ignore the intent log records by running "
7247 		    "'zpool clear'.\n"));
7248 		break;
7249 
7250 	case ZPOOL_STATUS_ERRATA:
7251 		(void) printf(gettext("status: Errata #%d detected.\n"),
7252 		    errata);
7253 
7254 		switch (errata) {
7255 		case ZPOOL_ERRATA_NONE:
7256 			break;
7257 
7258 		case ZPOOL_ERRATA_ZOL_2094_SCRUB:
7259 			(void) printf(gettext("action: To correct the issue "
7260 			    "run 'zpool scrub'.\n"));
7261 			break;
7262 
7263 		case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
7264 			(void) printf(gettext("\tExisting encrypted datasets "
7265 			    "contain an on-disk incompatibility\n\twhich "
7266 			    "needs to be corrected.\n"));
7267 			(void) printf(gettext("action: To correct the issue "
7268 			    "backup existing encrypted datasets to new\n\t"
7269 			    "encrypted datasets and destroy the old ones. "
7270 			    "'zfs mount -o ro' can\n\tbe used to temporarily "
7271 			    "mount existing encrypted datasets readonly.\n"));
7272 			break;
7273 
7274 		case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
7275 			(void) printf(gettext("\tExisting encrypted datasets "
7276 			    "contain an on-disk incompatibility\n\twhich "
7277 			    "needs to be corrected.\n"));
7278 			(void) printf(gettext("action: To correct the issue "
7279 			    "enable the bookmark_v2 feature and "
7280 			    "backup\n\tany existing encrypted datasets to "
7281 			    "new encrypted datasets and\n\tdestroy the old "
7282 			    "ones. If this pool does not contain any\n\t"
7283 			    "encrypted datasets, simply enable the "
7284 			    "bookmark_v2 feature\n"));
7285 			break;
7286 
7287 		default:
7288 			/*
7289 			 * All errata which allow the pool to be imported
7290 			 * must contain an action message.
7291 			 */
7292 			assert(0);
7293 		}
7294 		break;
7295 
7296 	default:
7297 		/*
7298 		 * The remaining errors can't actually be generated, yet.
7299 		 */
7300 		assert(reason == ZPOOL_STATUS_OK);
7301 	}
7302 
7303 	if (msgid != NULL)
7304 		(void) printf(gettext("   see: http://illumos.org/msg/%s\n"),
7305 		    msgid);
7306 
7307 	if (config != NULL) {
7308 		uint64_t nerr;
7309 		nvlist_t **spares, **l2cache;
7310 		uint_t nspares, nl2cache;
7311 		pool_checkpoint_stat_t *pcs = NULL;
7312 		pool_scan_stat_t *ps = NULL;
7313 		pool_removal_stat_t *prs = NULL;
7314 
7315 		(void) nvlist_lookup_uint64_array(nvroot,
7316 		    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
7317 		(void) nvlist_lookup_uint64_array(nvroot,
7318 		    ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &c);
7319 		(void) nvlist_lookup_uint64_array(nvroot,
7320 		    ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
7321 
7322 		print_scan_status(ps);
7323 		print_checkpoint_scan_warning(ps, pcs);
7324 		print_removal_status(zhp, prs);
7325 		print_checkpoint_status(pcs);
7326 
7327 		cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
7328 		    cbp->cb_name_flags);
7329 		if (cbp->cb_namewidth < 10)
7330 			cbp->cb_namewidth = 10;
7331 
7332 		(void) printf(gettext("config:\n\n"));
7333 		(void) printf(gettext("\t%-*s  %-8s %5s %5s %5s\n"),
7334 		    cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
7335 		    "CKSUM");
7336 
7337 		if (cbp->cb_print_slow_ios)
7338 			(void) printf(" %5s", gettext("SLOW"));
7339 
7340 		print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
7341 		    B_FALSE);
7342 
7343 		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_DEDUP);
7344 		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
7345 		print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_CLASS_LOGS);
7346 
7347 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
7348 		    &l2cache, &nl2cache) == 0)
7349 			print_l2cache(zhp, cbp, l2cache, nl2cache);
7350 
7351 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
7352 		    &spares, &nspares) == 0)
7353 			print_spares(zhp, cbp, spares, nspares);
7354 
7355 		if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
7356 		    &nerr) == 0) {
7357 			nvlist_t *nverrlist = NULL;
7358 
7359 			/*
7360 			 * If the approximate error count is small, get a
7361 			 * precise count by fetching the entire log and
7362 			 * uniquifying the results.
7363 			 */
7364 			if (nerr > 0 && nerr < 100 && !cbp->cb_verbose &&
7365 			    zpool_get_errlog(zhp, &nverrlist) == 0) {
7366 				nvpair_t *elem;
7367 
7368 				elem = NULL;
7369 				nerr = 0;
7370 				while ((elem = nvlist_next_nvpair(nverrlist,
7371 				    elem)) != NULL) {
7372 					nerr++;
7373 				}
7374 			}
7375 			nvlist_free(nverrlist);
7376 
7377 			(void) printf("\n");
7378 
7379 			if (nerr == 0)
7380 				(void) printf(gettext("errors: No known data "
7381 				    "errors\n"));
7382 			else if (!cbp->cb_verbose)
7383 				(void) printf(gettext("errors: %llu data "
7384 				    "errors, use '-v' for a list\n"),
7385 				    (u_longlong_t)nerr);
7386 			else
7387 				print_error_log(zhp);
7388 		}
7389 
7390 		if (cbp->cb_dedup_stats)
7391 			print_dedup_stats(config);
7392 	} else {
7393 		(void) printf(gettext("config: The configuration cannot be "
7394 		    "determined.\n"));
7395 	}
7396 
7397 	return (0);
7398 }
7399 
7400 /*
7401  * zpool status [-igLpPstvx] [-T d|u] [pool] ... [interval [count]]
7402  *
7403  *	-i	Display vdev initialization status.
7404  *	-g	Display guid for individual vdev name.
7405  *	-L	Follow links when resolving vdev path name.
7406  *	-p	Display values in parsable (exact) format.
7407  *	-P	Display full path for vdev name.
7408  *	-s	Display slow IOs column.
7409  *	-v	Display complete error logs
7410  *	-x	Display only pools with potential problems
7411  *	-D	Display dedup status (undocumented)
7412  *	-t	Display vdev TRIM status.
7413  *	-T	Display a timestamp in date(1) or Unix format
7414  *
7415  * Describes the health status of all pools or some subset.
7416  */
7417 int
7418 zpool_do_status(int argc, char **argv)
7419 {
7420 	int c;
7421 	int ret;
7422 	float interval = 0;
7423 	unsigned long count = 0;
7424 	status_cbdata_t cb = { 0 };
7425 
7426 	/* check options */
7427 	while ((c = getopt(argc, argv, "igLpPsvxDtT:")) != -1) {
7428 		switch (c) {
7429 		case 'i':
7430 			cb.cb_print_vdev_init = B_TRUE;
7431 			break;
7432 		case 'g':
7433 			cb.cb_name_flags |= VDEV_NAME_GUID;
7434 			break;
7435 		case 'L':
7436 			cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
7437 			break;
7438 		case 'p':
7439 			cb.cb_literal = B_TRUE;
7440 			break;
7441 		case 'P':
7442 			cb.cb_name_flags |= VDEV_NAME_PATH;
7443 			break;
7444 		case 's':
7445 			cb.cb_print_slow_ios = B_TRUE;
7446 			break;
7447 		case 'v':
7448 			cb.cb_verbose = B_TRUE;
7449 			break;
7450 		case 'x':
7451 			cb.cb_explain = B_TRUE;
7452 			break;
7453 		case 'D':
7454 			cb.cb_dedup_stats = B_TRUE;
7455 			break;
7456 		case 't':
7457 			cb.cb_print_vdev_trim = B_TRUE;
7458 			break;
7459 		case 'T':
7460 			get_timestamp_arg(*optarg);
7461 			break;
7462 		case '?':
7463 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
7464 			    optopt);
7465 			usage(B_FALSE);
7466 		}
7467 	}
7468 
7469 	argc -= optind;
7470 	argv += optind;
7471 
7472 	get_interval_count(&argc, argv, &interval, &count);
7473 
7474 	if (argc == 0)
7475 		cb.cb_allpools = B_TRUE;
7476 
7477 	cb.cb_first = B_TRUE;
7478 	cb.cb_print_status = B_TRUE;
7479 
7480 	for (;;) {
7481 		if (timestamp_fmt != NODATE)
7482 			print_timestamp(timestamp_fmt);
7483 
7484 		ret = for_each_pool(argc, argv, B_TRUE, NULL,
7485 		    status_callback, &cb);
7486 
7487 		if (argc == 0 && cb.cb_count == 0)
7488 			(void) printf(gettext("no pools available\n"));
7489 		else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
7490 			(void) printf(gettext("all pools are healthy\n"));
7491 
7492 		if (ret != 0)
7493 			return (ret);
7494 
7495 		if (interval == 0)
7496 			break;
7497 
7498 		if (count != 0 && --count == 0)
7499 			break;
7500 
7501 		(void) fsleep(interval);
7502 	}
7503 
7504 	return (0);
7505 }
7506 
7507 typedef struct upgrade_cbdata {
7508 	int	cb_first;
7509 	int	cb_argc;
7510 	uint64_t cb_version;
7511 	char	**cb_argv;
7512 } upgrade_cbdata_t;
7513 
7514 static int
7515 upgrade_version(zpool_handle_t *zhp, uint64_t version)
7516 {
7517 	int ret;
7518 	nvlist_t *config;
7519 	uint64_t oldversion;
7520 
7521 	config = zpool_get_config(zhp, NULL);
7522 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7523 	    &oldversion) == 0);
7524 
7525 	assert(SPA_VERSION_IS_SUPPORTED(oldversion));
7526 	assert(oldversion < version);
7527 
7528 	ret = zpool_upgrade(zhp, version);
7529 	if (ret != 0)
7530 		return (ret);
7531 
7532 	if (version >= SPA_VERSION_FEATURES) {
7533 		(void) printf(gettext("Successfully upgraded "
7534 		    "'%s' from version %llu to feature flags.\n"),
7535 		    zpool_get_name(zhp), oldversion);
7536 	} else {
7537 		(void) printf(gettext("Successfully upgraded "
7538 		    "'%s' from version %llu to version %llu.\n"),
7539 		    zpool_get_name(zhp), oldversion, version);
7540 	}
7541 
7542 	return (0);
7543 }
7544 
7545 static int
7546 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
7547 {
7548 	int i, ret, count;
7549 	boolean_t firstff = B_TRUE;
7550 	nvlist_t *enabled = zpool_get_features(zhp);
7551 
7552 	count = 0;
7553 	for (i = 0; i < SPA_FEATURES; i++) {
7554 		const char *fname = spa_feature_table[i].fi_uname;
7555 		const char *fguid = spa_feature_table[i].fi_guid;
7556 		if (!nvlist_exists(enabled, fguid)) {
7557 			char *propname;
7558 			verify(-1 != asprintf(&propname, "feature@%s", fname));
7559 			ret = zpool_set_prop(zhp, propname,
7560 			    ZFS_FEATURE_ENABLED);
7561 			if (ret != 0) {
7562 				free(propname);
7563 				return (ret);
7564 			}
7565 			count++;
7566 
7567 			if (firstff) {
7568 				(void) printf(gettext("Enabled the "
7569 				    "following features on '%s':\n"),
7570 				    zpool_get_name(zhp));
7571 				firstff = B_FALSE;
7572 			}
7573 			(void) printf(gettext("  %s\n"), fname);
7574 			free(propname);
7575 		}
7576 	}
7577 
7578 	if (countp != NULL)
7579 		*countp = count;
7580 	return (0);
7581 }
7582 
7583 static int
7584 upgrade_cb(zpool_handle_t *zhp, void *arg)
7585 {
7586 	upgrade_cbdata_t *cbp = arg;
7587 	nvlist_t *config;
7588 	uint64_t version;
7589 	boolean_t printnl = B_FALSE;
7590 	int ret;
7591 
7592 	config = zpool_get_config(zhp, NULL);
7593 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7594 	    &version) == 0);
7595 
7596 	assert(SPA_VERSION_IS_SUPPORTED(version));
7597 
7598 	if (version < cbp->cb_version) {
7599 		cbp->cb_first = B_FALSE;
7600 		ret = upgrade_version(zhp, cbp->cb_version);
7601 		if (ret != 0)
7602 			return (ret);
7603 		printnl = B_TRUE;
7604 
7605 		/*
7606 		 * If they did "zpool upgrade -a", then we could
7607 		 * be doing ioctls to different pools.  We need
7608 		 * to log this history once to each pool, and bypass
7609 		 * the normal history logging that happens in main().
7610 		 */
7611 		(void) zpool_log_history(g_zfs, history_str);
7612 		log_history = B_FALSE;
7613 	}
7614 
7615 	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
7616 		int count;
7617 		ret = upgrade_enable_all(zhp, &count);
7618 		if (ret != 0)
7619 			return (ret);
7620 
7621 		if (count > 0) {
7622 			cbp->cb_first = B_FALSE;
7623 			printnl = B_TRUE;
7624 		}
7625 	}
7626 
7627 	if (printnl) {
7628 		(void) printf(gettext("\n"));
7629 	}
7630 
7631 	return (0);
7632 }
7633 
7634 static int
7635 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
7636 {
7637 	upgrade_cbdata_t *cbp = arg;
7638 	nvlist_t *config;
7639 	uint64_t version;
7640 
7641 	config = zpool_get_config(zhp, NULL);
7642 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7643 	    &version) == 0);
7644 
7645 	assert(SPA_VERSION_IS_SUPPORTED(version));
7646 
7647 	if (version < SPA_VERSION_FEATURES) {
7648 		if (cbp->cb_first) {
7649 			(void) printf(gettext("The following pools are "
7650 			    "formatted with legacy version numbers and can\n"
7651 			    "be upgraded to use feature flags.  After "
7652 			    "being upgraded, these pools\nwill no "
7653 			    "longer be accessible by software that does not "
7654 			    "support feature\nflags.\n\n"));
7655 			(void) printf(gettext("VER  POOL\n"));
7656 			(void) printf(gettext("---  ------------\n"));
7657 			cbp->cb_first = B_FALSE;
7658 		}
7659 
7660 		(void) printf("%2llu   %s\n", (u_longlong_t)version,
7661 		    zpool_get_name(zhp));
7662 	}
7663 
7664 	return (0);
7665 }
7666 
7667 static int
7668 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
7669 {
7670 	upgrade_cbdata_t *cbp = arg;
7671 	nvlist_t *config;
7672 	uint64_t version;
7673 
7674 	config = zpool_get_config(zhp, NULL);
7675 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7676 	    &version) == 0);
7677 
7678 	if (version >= SPA_VERSION_FEATURES) {
7679 		int i;
7680 		boolean_t poolfirst = B_TRUE;
7681 		nvlist_t *enabled = zpool_get_features(zhp);
7682 
7683 		for (i = 0; i < SPA_FEATURES; i++) {
7684 			const char *fguid = spa_feature_table[i].fi_guid;
7685 			const char *fname = spa_feature_table[i].fi_uname;
7686 			if (!nvlist_exists(enabled, fguid)) {
7687 				if (cbp->cb_first) {
7688 					(void) printf(gettext("\nSome "
7689 					    "supported features are not "
7690 					    "enabled on the following pools. "
7691 					    "Once a\nfeature is enabled the "
7692 					    "pool may become incompatible with "
7693 					    "software\nthat does not support "
7694 					    "the feature. See "
7695 					    "zpool-features(5) for "
7696 					    "details.\n\n"));
7697 					(void) printf(gettext("POOL  "
7698 					    "FEATURE\n"));
7699 					(void) printf(gettext("------"
7700 					    "---------\n"));
7701 					cbp->cb_first = B_FALSE;
7702 				}
7703 
7704 				if (poolfirst) {
7705 					(void) printf(gettext("%s\n"),
7706 					    zpool_get_name(zhp));
7707 					poolfirst = B_FALSE;
7708 				}
7709 
7710 				(void) printf(gettext("      %s\n"), fname);
7711 			}
7712 		}
7713 	}
7714 
7715 	return (0);
7716 }
7717 
7718 /* ARGSUSED */
7719 static int
7720 upgrade_one(zpool_handle_t *zhp, void *data)
7721 {
7722 	boolean_t printnl = B_FALSE;
7723 	upgrade_cbdata_t *cbp = data;
7724 	uint64_t cur_version;
7725 	int ret;
7726 
7727 	if (strcmp("log", zpool_get_name(zhp)) == 0) {
7728 		(void) printf(gettext("'log' is now a reserved word\n"
7729 		    "Pool 'log' must be renamed using export and import"
7730 		    " to upgrade.\n"));
7731 		return (1);
7732 	}
7733 
7734 	cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
7735 	if (cur_version > cbp->cb_version) {
7736 		(void) printf(gettext("Pool '%s' is already formatted "
7737 		    "using more current version '%llu'.\n\n"),
7738 		    zpool_get_name(zhp), cur_version);
7739 		return (0);
7740 	}
7741 
7742 	if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
7743 		(void) printf(gettext("Pool '%s' is already formatted "
7744 		    "using version %llu.\n\n"), zpool_get_name(zhp),
7745 		    cbp->cb_version);
7746 		return (0);
7747 	}
7748 
7749 	if (cur_version != cbp->cb_version) {
7750 		printnl = B_TRUE;
7751 		ret = upgrade_version(zhp, cbp->cb_version);
7752 		if (ret != 0)
7753 			return (ret);
7754 	}
7755 
7756 	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
7757 		int count = 0;
7758 		ret = upgrade_enable_all(zhp, &count);
7759 		if (ret != 0)
7760 			return (ret);
7761 
7762 		if (count != 0) {
7763 			printnl = B_TRUE;
7764 		} else if (cur_version == SPA_VERSION) {
7765 			(void) printf(gettext("Pool '%s' already has all "
7766 			    "supported features enabled.\n"),
7767 			    zpool_get_name(zhp));
7768 		}
7769 	}
7770 
7771 	if (printnl) {
7772 		(void) printf(gettext("\n"));
7773 	}
7774 
7775 	return (0);
7776 }
7777 
7778 /*
7779  * zpool upgrade
7780  * zpool upgrade -v
7781  * zpool upgrade [-V version] <-a | pool ...>
7782  *
7783  * With no arguments, display downrev'd ZFS pool available for upgrade.
7784  * Individual pools can be upgraded by specifying the pool, and '-a' will
7785  * upgrade all pools.
7786  */
7787 int
7788 zpool_do_upgrade(int argc, char **argv)
7789 {
7790 	int c;
7791 	upgrade_cbdata_t cb = { 0 };
7792 	int ret = 0;
7793 	boolean_t showversions = B_FALSE;
7794 	boolean_t upgradeall = B_FALSE;
7795 	char *end;
7796 
7797 
7798 	/* check options */
7799 	while ((c = getopt(argc, argv, ":avV:")) != -1) {
7800 		switch (c) {
7801 		case 'a':
7802 			upgradeall = B_TRUE;
7803 			break;
7804 		case 'v':
7805 			showversions = B_TRUE;
7806 			break;
7807 		case 'V':
7808 			cb.cb_version = strtoll(optarg, &end, 10);
7809 			if (*end != '\0' ||
7810 			    !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
7811 				(void) fprintf(stderr,
7812 				    gettext("invalid version '%s'\n"), optarg);
7813 				usage(B_FALSE);
7814 			}
7815 			break;
7816 		case ':':
7817 			(void) fprintf(stderr, gettext("missing argument for "
7818 			    "'%c' option\n"), optopt);
7819 			usage(B_FALSE);
7820 			break;
7821 		case '?':
7822 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
7823 			    optopt);
7824 			usage(B_FALSE);
7825 		}
7826 	}
7827 
7828 	cb.cb_argc = argc;
7829 	cb.cb_argv = argv;
7830 	argc -= optind;
7831 	argv += optind;
7832 
7833 	if (cb.cb_version == 0) {
7834 		cb.cb_version = SPA_VERSION;
7835 	} else if (!upgradeall && argc == 0) {
7836 		(void) fprintf(stderr, gettext("-V option is "
7837 		    "incompatible with other arguments\n"));
7838 		usage(B_FALSE);
7839 	}
7840 
7841 	if (showversions) {
7842 		if (upgradeall || argc != 0) {
7843 			(void) fprintf(stderr, gettext("-v option is "
7844 			    "incompatible with other arguments\n"));
7845 			usage(B_FALSE);
7846 		}
7847 	} else if (upgradeall) {
7848 		if (argc != 0) {
7849 			(void) fprintf(stderr, gettext("-a option should not "
7850 			    "be used along with a pool name\n"));
7851 			usage(B_FALSE);
7852 		}
7853 	}
7854 
7855 	(void) printf(gettext("This system supports ZFS pool feature "
7856 	    "flags.\n\n"));
7857 	if (showversions) {
7858 		int i;
7859 
7860 		(void) printf(gettext("The following features are "
7861 		    "supported:\n\n"));
7862 		(void) printf(gettext("FEAT DESCRIPTION\n"));
7863 		(void) printf("----------------------------------------------"
7864 		    "---------------\n");
7865 		for (i = 0; i < SPA_FEATURES; i++) {
7866 			zfeature_info_t *fi = &spa_feature_table[i];
7867 			const char *ro =
7868 			    (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
7869 			    " (read-only compatible)" : "";
7870 
7871 			(void) printf("%-37s%s\n", fi->fi_uname, ro);
7872 			(void) printf("     %s\n", fi->fi_desc);
7873 		}
7874 		(void) printf("\n");
7875 
7876 		(void) printf(gettext("The following legacy versions are also "
7877 		    "supported:\n\n"));
7878 		(void) printf(gettext("VER  DESCRIPTION\n"));
7879 		(void) printf("---  -----------------------------------------"
7880 		    "---------------\n");
7881 		(void) printf(gettext(" 1   Initial ZFS version\n"));
7882 		(void) printf(gettext(" 2   Ditto blocks "
7883 		    "(replicated metadata)\n"));
7884 		(void) printf(gettext(" 3   Hot spares and double parity "
7885 		    "RAID-Z\n"));
7886 		(void) printf(gettext(" 4   zpool history\n"));
7887 		(void) printf(gettext(" 5   Compression using the gzip "
7888 		    "algorithm\n"));
7889 		(void) printf(gettext(" 6   bootfs pool property\n"));
7890 		(void) printf(gettext(" 7   Separate intent log devices\n"));
7891 		(void) printf(gettext(" 8   Delegated administration\n"));
7892 		(void) printf(gettext(" 9   refquota and refreservation "
7893 		    "properties\n"));
7894 		(void) printf(gettext(" 10  Cache devices\n"));
7895 		(void) printf(gettext(" 11  Improved scrub performance\n"));
7896 		(void) printf(gettext(" 12  Snapshot properties\n"));
7897 		(void) printf(gettext(" 13  snapused property\n"));
7898 		(void) printf(gettext(" 14  passthrough-x aclinherit\n"));
7899 		(void) printf(gettext(" 15  user/group space accounting\n"));
7900 		(void) printf(gettext(" 16  stmf property support\n"));
7901 		(void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
7902 		(void) printf(gettext(" 18  Snapshot user holds\n"));
7903 		(void) printf(gettext(" 19  Log device removal\n"));
7904 		(void) printf(gettext(" 20  Compression using zle "
7905 		    "(zero-length encoding)\n"));
7906 		(void) printf(gettext(" 21  Deduplication\n"));
7907 		(void) printf(gettext(" 22  Received properties\n"));
7908 		(void) printf(gettext(" 23  Slim ZIL\n"));
7909 		(void) printf(gettext(" 24  System attributes\n"));
7910 		(void) printf(gettext(" 25  Improved scrub stats\n"));
7911 		(void) printf(gettext(" 26  Improved snapshot deletion "
7912 		    "performance\n"));
7913 		(void) printf(gettext(" 27  Improved snapshot creation "
7914 		    "performance\n"));
7915 		(void) printf(gettext(" 28  Multiple vdev replacements\n"));
7916 		(void) printf(gettext("\nFor more information on a particular "
7917 		    "version, including supported releases,\n"));
7918 		(void) printf(gettext("see the ZFS Administration Guide.\n\n"));
7919 	} else if (argc == 0 && upgradeall) {
7920 		cb.cb_first = B_TRUE;
7921 		ret = zpool_iter(g_zfs, upgrade_cb, &cb);
7922 		if (ret == 0 && cb.cb_first) {
7923 			if (cb.cb_version == SPA_VERSION) {
7924 				(void) printf(gettext("All pools are already "
7925 				    "formatted using feature flags.\n\n"));
7926 				(void) printf(gettext("Every feature flags "
7927 				    "pool already has all supported features "
7928 				    "enabled.\n"));
7929 			} else {
7930 				(void) printf(gettext("All pools are already "
7931 				    "formatted with version %llu or higher.\n"),
7932 				    cb.cb_version);
7933 			}
7934 		}
7935 	} else if (argc == 0) {
7936 		cb.cb_first = B_TRUE;
7937 		ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
7938 		assert(ret == 0);
7939 
7940 		if (cb.cb_first) {
7941 			(void) printf(gettext("All pools are formatted "
7942 			    "using feature flags.\n\n"));
7943 		} else {
7944 			(void) printf(gettext("\nUse 'zpool upgrade -v' "
7945 			    "for a list of available legacy versions.\n"));
7946 		}
7947 
7948 		cb.cb_first = B_TRUE;
7949 		ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
7950 		assert(ret == 0);
7951 
7952 		if (cb.cb_first) {
7953 			(void) printf(gettext("Every feature flags pool has "
7954 			    "all supported features enabled.\n"));
7955 		} else {
7956 			(void) printf(gettext("\n"));
7957 		}
7958 	} else {
7959 		ret = for_each_pool(argc, argv, B_FALSE, NULL,
7960 		    upgrade_one, &cb);
7961 	}
7962 
7963 	return (ret);
7964 }
7965 
7966 typedef struct hist_cbdata {
7967 	boolean_t first;
7968 	boolean_t longfmt;
7969 	boolean_t internal;
7970 } hist_cbdata_t;
7971 
7972 static void
7973 print_history_records(nvlist_t *nvhis, hist_cbdata_t *cb)
7974 {
7975 	nvlist_t **records;
7976 	uint_t numrecords;
7977 	int i;
7978 
7979 	verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
7980 	    &records, &numrecords) == 0);
7981 	for (i = 0; i < numrecords; i++) {
7982 		nvlist_t *rec = records[i];
7983 		char tbuf[30] = "";
7984 
7985 		if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
7986 			time_t tsec;
7987 			struct tm t;
7988 
7989 			tsec = fnvlist_lookup_uint64(records[i],
7990 			    ZPOOL_HIST_TIME);
7991 			(void) localtime_r(&tsec, &t);
7992 			(void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
7993 		}
7994 
7995 		if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
7996 			(void) printf("%s %s", tbuf,
7997 			    fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
7998 		} else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
7999 			int ievent =
8000 			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
8001 			if (!cb->internal)
8002 				continue;
8003 			if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
8004 				(void) printf("%s unrecognized record:\n",
8005 				    tbuf);
8006 				dump_nvlist(rec, 4);
8007 				continue;
8008 			}
8009 			(void) printf("%s [internal %s txg:%lld] %s", tbuf,
8010 			    zfs_history_event_names[ievent],
8011 			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_TXG),
8012 			    fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
8013 		} else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
8014 			if (!cb->internal)
8015 				continue;
8016 			(void) printf("%s [txg:%lld] %s", tbuf,
8017 			    fnvlist_lookup_uint64(rec, ZPOOL_HIST_TXG),
8018 			    fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
8019 			if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
8020 				(void) printf(" %s (%llu)",
8021 				    fnvlist_lookup_string(rec,
8022 				    ZPOOL_HIST_DSNAME),
8023 				    fnvlist_lookup_uint64(rec,
8024 				    ZPOOL_HIST_DSID));
8025 			}
8026 			(void) printf(" %s", fnvlist_lookup_string(rec,
8027 			    ZPOOL_HIST_INT_STR));
8028 		} else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
8029 			if (!cb->internal)
8030 				continue;
8031 			(void) printf("%s ioctl %s\n", tbuf,
8032 			    fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
8033 			if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
8034 				(void) printf("    input:\n");
8035 				dump_nvlist(fnvlist_lookup_nvlist(rec,
8036 				    ZPOOL_HIST_INPUT_NVL), 8);
8037 			}
8038 			if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
8039 				(void) printf("    output:\n");
8040 				dump_nvlist(fnvlist_lookup_nvlist(rec,
8041 				    ZPOOL_HIST_OUTPUT_NVL), 8);
8042 			}
8043 			if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
8044 				(void) printf("    errno: %lld\n",
8045 				    fnvlist_lookup_int64(rec,
8046 				    ZPOOL_HIST_ERRNO));
8047 			}
8048 		} else {
8049 			if (!cb->internal)
8050 				continue;
8051 			(void) printf("%s unrecognized record:\n", tbuf);
8052 			dump_nvlist(rec, 4);
8053 		}
8054 
8055 		if (!cb->longfmt) {
8056 			(void) printf("\n");
8057 			continue;
8058 		}
8059 		(void) printf(" [");
8060 		if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
8061 			uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
8062 			struct passwd *pwd = getpwuid(who);
8063 			(void) printf("user %d ", (int)who);
8064 			if (pwd != NULL)
8065 				(void) printf("(%s) ", pwd->pw_name);
8066 		}
8067 		if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
8068 			(void) printf("on %s",
8069 			    fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
8070 		}
8071 		if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
8072 			(void) printf(":%s",
8073 			    fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
8074 		}
8075 		(void) printf("]");
8076 		(void) printf("\n");
8077 	}
8078 }
8079 
8080 /*
8081  * Print out the command history for a specific pool.
8082  */
8083 static int
8084 get_history_one(zpool_handle_t *zhp, void *data)
8085 {
8086 	nvlist_t *nvhis;
8087 	int ret;
8088 	hist_cbdata_t *cb = (hist_cbdata_t *)data;
8089 	uint64_t off = 0;
8090 	boolean_t eof = B_FALSE;
8091 
8092 	cb->first = B_FALSE;
8093 
8094 	(void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
8095 
8096 	while (!eof) {
8097 		if ((ret = zpool_get_history(zhp, &nvhis, &off, &eof)) != 0)
8098 			return (ret);
8099 
8100 		print_history_records(nvhis, cb);
8101 		nvlist_free(nvhis);
8102 	}
8103 	(void) printf("\n");
8104 
8105 	return (ret);
8106 }
8107 
8108 /*
8109  * zpool history <pool>
8110  *
8111  * Displays the history of commands that modified pools.
8112  */
8113 int
8114 zpool_do_history(int argc, char **argv)
8115 {
8116 	hist_cbdata_t cbdata = { 0 };
8117 	int ret;
8118 	int c;
8119 
8120 	cbdata.first = B_TRUE;
8121 	/* check options */
8122 	while ((c = getopt(argc, argv, "li")) != -1) {
8123 		switch (c) {
8124 		case 'l':
8125 			cbdata.longfmt = B_TRUE;
8126 			break;
8127 		case 'i':
8128 			cbdata.internal = B_TRUE;
8129 			break;
8130 		case '?':
8131 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
8132 			    optopt);
8133 			usage(B_FALSE);
8134 		}
8135 	}
8136 	argc -= optind;
8137 	argv += optind;
8138 
8139 	ret = for_each_pool(argc, argv, B_FALSE,  NULL, get_history_one,
8140 	    &cbdata);
8141 
8142 	if (argc == 0 && cbdata.first == B_TRUE) {
8143 		(void) printf(gettext("no pools available\n"));
8144 		return (0);
8145 	}
8146 
8147 	return (ret);
8148 }
8149 
8150 static int
8151 get_callback(zpool_handle_t *zhp, void *data)
8152 {
8153 	zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
8154 	char value[MAXNAMELEN];
8155 	zprop_source_t srctype;
8156 	zprop_list_t *pl;
8157 
8158 	for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
8159 
8160 		/*
8161 		 * Skip the special fake placeholder. This will also skip
8162 		 * over the name property when 'all' is specified.
8163 		 */
8164 		if (pl->pl_prop == ZPOOL_PROP_NAME &&
8165 		    pl == cbp->cb_proplist)
8166 			continue;
8167 
8168 		if (pl->pl_prop == ZPROP_INVAL &&
8169 		    (zpool_prop_feature(pl->pl_user_prop) ||
8170 		    zpool_prop_unsupported(pl->pl_user_prop))) {
8171 			srctype = ZPROP_SRC_LOCAL;
8172 
8173 			if (zpool_prop_get_feature(zhp, pl->pl_user_prop,
8174 			    value, sizeof (value)) == 0) {
8175 				zprop_print_one_property(zpool_get_name(zhp),
8176 				    cbp, pl->pl_user_prop, value, srctype,
8177 				    NULL, NULL);
8178 			}
8179 		} else {
8180 			if (zpool_get_prop(zhp, pl->pl_prop, value,
8181 			    sizeof (value), &srctype, cbp->cb_literal) != 0)
8182 				continue;
8183 
8184 			zprop_print_one_property(zpool_get_name(zhp), cbp,
8185 			    zpool_prop_to_name(pl->pl_prop), value, srctype,
8186 			    NULL, NULL);
8187 		}
8188 	}
8189 	return (0);
8190 }
8191 
8192 /*
8193  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
8194  *
8195  *	-H	Scripted mode.  Don't display headers, and separate properties
8196  *		by a single tab.
8197  *	-o	List of columns to display.  Defaults to
8198  *		"name,property,value,source".
8199  *	-p	Diplay values in parsable (exact) format.
8200  *
8201  * Get properties of pools in the system. Output space statistics
8202  * for each one as well as other attributes.
8203  */
8204 int
8205 zpool_do_get(int argc, char **argv)
8206 {
8207 	zprop_get_cbdata_t cb = { 0 };
8208 	zprop_list_t fake_name = { 0 };
8209 	int ret;
8210 	int c, i;
8211 	char *value;
8212 
8213 	cb.cb_first = B_TRUE;
8214 
8215 	/*
8216 	 * Set up default columns and sources.
8217 	 */
8218 	cb.cb_sources = ZPROP_SRC_ALL;
8219 	cb.cb_columns[0] = GET_COL_NAME;
8220 	cb.cb_columns[1] = GET_COL_PROPERTY;
8221 	cb.cb_columns[2] = GET_COL_VALUE;
8222 	cb.cb_columns[3] = GET_COL_SOURCE;
8223 	cb.cb_type = ZFS_TYPE_POOL;
8224 
8225 	/* check options */
8226 	while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
8227 		switch (c) {
8228 		case 'p':
8229 			cb.cb_literal = B_TRUE;
8230 			break;
8231 		case 'H':
8232 			cb.cb_scripted = B_TRUE;
8233 			break;
8234 		case 'o':
8235 			bzero(&cb.cb_columns, sizeof (cb.cb_columns));
8236 			i = 0;
8237 			while (*optarg != '\0') {
8238 				static char *col_subopts[] =
8239 				{ "name", "property", "value", "source",
8240 				"all", NULL };
8241 
8242 				if (i == ZFS_GET_NCOLS) {
8243 					(void) fprintf(stderr, gettext("too "
8244 					"many fields given to -o "
8245 					"option\n"));
8246 					usage(B_FALSE);
8247 				}
8248 
8249 				switch (getsubopt(&optarg, col_subopts,
8250 				    &value)) {
8251 				case 0:
8252 					cb.cb_columns[i++] = GET_COL_NAME;
8253 					break;
8254 				case 1:
8255 					cb.cb_columns[i++] = GET_COL_PROPERTY;
8256 					break;
8257 				case 2:
8258 					cb.cb_columns[i++] = GET_COL_VALUE;
8259 					break;
8260 				case 3:
8261 					cb.cb_columns[i++] = GET_COL_SOURCE;
8262 					break;
8263 				case 4:
8264 					if (i > 0) {
8265 						(void) fprintf(stderr,
8266 						    gettext("\"all\" conflicts "
8267 						    "with specific fields "
8268 						    "given to -o option\n"));
8269 						usage(B_FALSE);
8270 					}
8271 					cb.cb_columns[0] = GET_COL_NAME;
8272 					cb.cb_columns[1] = GET_COL_PROPERTY;
8273 					cb.cb_columns[2] = GET_COL_VALUE;
8274 					cb.cb_columns[3] = GET_COL_SOURCE;
8275 					i = ZFS_GET_NCOLS;
8276 					break;
8277 				default:
8278 					(void) fprintf(stderr,
8279 					    gettext("invalid column name "
8280 					    "'%s'\n"), value);
8281 					usage(B_FALSE);
8282 				}
8283 			}
8284 			break;
8285 		case '?':
8286 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
8287 			    optopt);
8288 			usage(B_FALSE);
8289 		}
8290 	}
8291 
8292 	argc -= optind;
8293 	argv += optind;
8294 
8295 	if (argc < 1) {
8296 		(void) fprintf(stderr, gettext("missing property "
8297 		    "argument\n"));
8298 		usage(B_FALSE);
8299 	}
8300 
8301 	if (zprop_get_list(g_zfs, argv[0], &cb.cb_proplist,
8302 	    ZFS_TYPE_POOL) != 0)
8303 		usage(B_FALSE);
8304 
8305 	argc--;
8306 	argv++;
8307 
8308 	if (cb.cb_proplist != NULL) {
8309 		fake_name.pl_prop = ZPOOL_PROP_NAME;
8310 		fake_name.pl_width = strlen(gettext("NAME"));
8311 		fake_name.pl_next = cb.cb_proplist;
8312 		cb.cb_proplist = &fake_name;
8313 	}
8314 
8315 	ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist,
8316 	    get_callback, &cb);
8317 
8318 	if (cb.cb_proplist == &fake_name)
8319 		zprop_free_list(fake_name.pl_next);
8320 	else
8321 		zprop_free_list(cb.cb_proplist);
8322 
8323 	return (ret);
8324 }
8325 
8326 typedef struct set_cbdata {
8327 	char *cb_propname;
8328 	char *cb_value;
8329 	boolean_t cb_any_successful;
8330 } set_cbdata_t;
8331 
8332 int
8333 set_callback(zpool_handle_t *zhp, void *data)
8334 {
8335 	int error;
8336 	set_cbdata_t *cb = (set_cbdata_t *)data;
8337 
8338 	error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
8339 
8340 	if (!error)
8341 		cb->cb_any_successful = B_TRUE;
8342 
8343 	return (error);
8344 }
8345 
8346 int
8347 zpool_do_set(int argc, char **argv)
8348 {
8349 	set_cbdata_t cb = { 0 };
8350 	int error;
8351 
8352 	if (argc > 1 && argv[1][0] == '-') {
8353 		(void) fprintf(stderr, gettext("invalid option '%c'\n"),
8354 		    argv[1][1]);
8355 		usage(B_FALSE);
8356 	}
8357 
8358 	if (argc < 2) {
8359 		(void) fprintf(stderr, gettext("missing property=value "
8360 		    "argument\n"));
8361 		usage(B_FALSE);
8362 	}
8363 
8364 	if (argc < 3) {
8365 		(void) fprintf(stderr, gettext("missing pool name\n"));
8366 		usage(B_FALSE);
8367 	}
8368 
8369 	if (argc > 3) {
8370 		(void) fprintf(stderr, gettext("too many pool names\n"));
8371 		usage(B_FALSE);
8372 	}
8373 
8374 	cb.cb_propname = argv[1];
8375 	cb.cb_value = strchr(cb.cb_propname, '=');
8376 	if (cb.cb_value == NULL) {
8377 		(void) fprintf(stderr, gettext("missing value in "
8378 		    "property=value argument\n"));
8379 		usage(B_FALSE);
8380 	}
8381 
8382 	*(cb.cb_value) = '\0';
8383 	cb.cb_value++;
8384 
8385 	error = for_each_pool(argc - 2, argv + 2, B_TRUE, NULL,
8386 	    set_callback, &cb);
8387 
8388 	return (error);
8389 }
8390 
8391 static int
8392 find_command_idx(char *command, int *idx)
8393 {
8394 	int i;
8395 
8396 	for (i = 0; i < NCOMMAND; i++) {
8397 		if (command_table[i].name == NULL)
8398 			continue;
8399 
8400 		if (strcmp(command, command_table[i].name) == 0) {
8401 			*idx = i;
8402 			return (0);
8403 		}
8404 	}
8405 	return (1);
8406 }
8407 
8408 int
8409 main(int argc, char **argv)
8410 {
8411 	int ret = 0;
8412 	int i;
8413 	char *cmdname;
8414 
8415 	(void) setlocale(LC_ALL, "");
8416 	(void) textdomain(TEXT_DOMAIN);
8417 
8418 	if ((g_zfs = libzfs_init()) == NULL) {
8419 		(void) fprintf(stderr, gettext("internal error: failed to "
8420 		    "initialize ZFS library\n"));
8421 		return (1);
8422 	}
8423 
8424 	libzfs_print_on_error(g_zfs, B_TRUE);
8425 
8426 	opterr = 0;
8427 
8428 	/*
8429 	 * Make sure the user has specified some command.
8430 	 */
8431 	if (argc < 2) {
8432 		(void) fprintf(stderr, gettext("missing command\n"));
8433 		usage(B_FALSE);
8434 	}
8435 
8436 	cmdname = argv[1];
8437 
8438 	/*
8439 	 * Special case '-?'
8440 	 */
8441 	if (strcmp(cmdname, "-?") == 0)
8442 		usage(B_TRUE);
8443 
8444 	zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
8445 
8446 	/*
8447 	 * Run the appropriate command.
8448 	 */
8449 	if (find_command_idx(cmdname, &i) == 0) {
8450 		current_command = &command_table[i];
8451 		ret = command_table[i].func(argc - 1, argv + 1);
8452 	} else if (strchr(cmdname, '=')) {
8453 		verify(find_command_idx("set", &i) == 0);
8454 		current_command = &command_table[i];
8455 		ret = command_table[i].func(argc, argv);
8456 	} else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
8457 		/*
8458 		 * 'freeze' is a vile debugging abomination, so we treat
8459 		 * it as such.
8460 		 */
8461 		char buf[16384];
8462 		int fd = open(ZFS_DEV, O_RDWR);
8463 		(void) strcpy((void *)buf, argv[2]);
8464 		return (!!ioctl(fd, ZFS_IOC_POOL_FREEZE, buf));
8465 	} else {
8466 		(void) fprintf(stderr, gettext("unrecognized "
8467 		    "command '%s'\n"), cmdname);
8468 		usage(B_FALSE);
8469 	}
8470 
8471 	if (ret == 0 && log_history)
8472 		(void) zpool_log_history(g_zfs, history_str);
8473 
8474 	libzfs_fini(g_zfs);
8475 
8476 	/*
8477 	 * The 'ZFS_ABORT' environment variable causes us to dump core on exit
8478 	 * for the purposes of running ::findleaks.
8479 	 */
8480 	if (getenv("ZFS_ABORT") != NULL) {
8481 		(void) printf("dumping core by request\n");
8482 		abort();
8483 	}
8484 
8485 	return (ret);
8486 }
8487