xref: /titanic_51/usr/src/cmd/zinject/zinject.c (revision 48bc00d6814e04ff3edb32cafe7d1bc580baff68)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * ZFS Fault Injector
28  *
29  * This userland component takes a set of options and uses libzpool to translate
30  * from a user-visible object type and name to an internal representation.
31  * There are two basic types of faults: device faults and data faults.
32  *
33  *
34  * DEVICE FAULTS
35  *
36  * Errors can be injected into a particular vdev using the '-d' option.  This
37  * option takes a path or vdev GUID to uniquely identify the device within a
38  * pool.  There are two types of errors that can be injected, EIO and ENXIO,
39  * that can be controlled through the '-e' option.  The default is ENXIO.  For
40  * EIO failures, any attempt to read data from the device will return EIO, but
41  * subsequent attempt to reopen the device will succeed.  For ENXIO failures,
42  * any attempt to read from the device will return EIO, but any attempt to
43  * reopen the device will also return ENXIO.
44  * For label faults, the -L option must be specified. This allows faults
45  * to be injected into either the nvlist or uberblock region of all the labels
46  * for the specified device.
47  *
48  * This form of the command looks like:
49  *
50  * 	zinject -d device [-e errno] [-L <uber | nvlist>] pool
51  *
52  *
53  * DATA FAULTS
54  *
55  * We begin with a tuple of the form:
56  *
57  * 	<type,level,range,object>
58  *
59  * 	type	A string describing the type of data to target.  Each type
60  * 		implicitly describes how to interpret 'object'. Currently,
61  * 		the following values are supported:
62  *
63  * 		data		User data for a file
64  * 		dnode		Dnode for a file or directory
65  *
66  *		The following MOS objects are special.  Instead of injecting
67  *		errors on a particular object or blkid, we inject errors across
68  *		all objects of the given type.
69  *
70  * 		mos		Any data in the MOS
71  * 		mosdir		object directory
72  * 		config		pool configuration
73  * 		bplist		blkptr list
74  * 		spacemap	spacemap
75  * 		metaslab	metaslab
76  * 		errlog		persistent error log
77  *
78  * 	level	Object level.  Defaults to '0', not applicable to all types.  If
79  * 		a range is given, this corresponds to the indirect block
80  * 		corresponding to the specific range.
81  *
82  *	range	A numerical range [start,end) within the object.  Defaults to
83  *		the full size of the file.
84  *
85  * 	object	A string describing the logical location of the object.  For
86  * 		files and directories (currently the only supported types),
87  * 		this is the path of the object on disk.
88  *
89  * This is translated, via libzpool, into the following internal representation:
90  *
91  * 	<type,objset,object,level,range>
92  *
93  * These types should be self-explanatory.  This tuple is then passed to the
94  * kernel via a special ioctl() to initiate fault injection for the given
95  * object.  Note that 'type' is not strictly necessary for fault injection, but
96  * is used when translating existing faults into a human-readable string.
97  *
98  *
99  * The command itself takes one of the forms:
100  *
101  * 	zinject
102  * 	zinject <-a | -u pool>
103  * 	zinject -c <id|all>
104  * 	zinject [-q] <-t type> [-f freq] [-u] [-a] [-m] [-e errno] [-l level]
105  *	    [-r range] <object>
106  * 	zinject [-f freq] [-a] [-m] [-u] -b objset:object:level:start:end pool
107  *
108  * With no arguments, the command prints all currently registered injection
109  * handlers, with their numeric identifiers.
110  *
111  * The '-c' option will clear the given handler, or all handlers if 'all' is
112  * specified.
113  *
114  * The '-e' option takes a string describing the errno to simulate.  This must
115  * be either 'io' or 'checksum'.  In most cases this will result in the same
116  * behavior, but RAID-Z will produce a different set of ereports for this
117  * situation.
118  *
119  * The '-a', '-u', and '-m' flags toggle internal flush behavior.  If '-a' is
120  * specified, then the ARC cache is flushed appropriately.  If '-u' is
121  * specified, then the underlying SPA is unloaded.  Either of these flags can be
122  * specified independently of any other handlers.  The '-m' flag automatically
123  * does an unmount and remount of the underlying dataset to aid in flushing the
124  * cache.
125  *
126  * The '-f' flag controls the frequency of errors injected, expressed as a
127  * integer percentage between 1 and 100.  The default is 100.
128  *
129  * The this form is responsible for actually injecting the handler into the
130  * framework.  It takes the arguments described above, translates them to the
131  * internal tuple using libzpool, and then issues an ioctl() to register the
132  * handler.
133  *
134  * The final form can target a specific bookmark, regardless of whether a
135  * human-readable interface has been designed.  It allows developers to specify
136  * a particular block by number.
137  */
138 
139 #include <errno.h>
140 #include <fcntl.h>
141 #include <stdio.h>
142 #include <stdlib.h>
143 #include <strings.h>
144 #include <unistd.h>
145 
146 #include <sys/fs/zfs.h>
147 #include <sys/mount.h>
148 
149 #include <libzfs.h>
150 
151 #undef verify	/* both libzfs.h and zfs_context.h want to define this */
152 
153 #include "zinject.h"
154 
155 libzfs_handle_t *g_zfs;
156 int zfs_fd;
157 
158 #define	ECKSUM	EBADE
159 
160 static const char *errtable[TYPE_INVAL] = {
161 	"data",
162 	"dnode",
163 	"mos",
164 	"mosdir",
165 	"metaslab",
166 	"config",
167 	"bplist",
168 	"spacemap",
169 	"errlog",
170 	"uber",
171 	"nvlist"
172 };
173 
174 static err_type_t
175 name_to_type(const char *arg)
176 {
177 	int i;
178 	for (i = 0; i < TYPE_INVAL; i++)
179 		if (strcmp(errtable[i], arg) == 0)
180 			return (i);
181 
182 	return (TYPE_INVAL);
183 }
184 
185 static const char *
186 type_to_name(uint64_t type)
187 {
188 	switch (type) {
189 	case DMU_OT_OBJECT_DIRECTORY:
190 		return ("mosdir");
191 	case DMU_OT_OBJECT_ARRAY:
192 		return ("metaslab");
193 	case DMU_OT_PACKED_NVLIST:
194 		return ("config");
195 	case DMU_OT_BPLIST:
196 		return ("bplist");
197 	case DMU_OT_SPACE_MAP:
198 		return ("spacemap");
199 	case DMU_OT_ERROR_LOG:
200 		return ("errlog");
201 	default:
202 		return ("-");
203 	}
204 }
205 
206 
207 /*
208  * Print usage message.
209  */
210 void
211 usage(void)
212 {
213 	(void) printf(
214 	    "usage:\n"
215 	    "\n"
216 	    "\tzinject\n"
217 	    "\n"
218 	    "\t\tList all active injection records.\n"
219 	    "\n"
220 	    "\tzinject -c <id|all>\n"
221 	    "\n"
222 	    "\t\tClear the particular record (if given a numeric ID), or\n"
223 	    "\t\tall records if 'all' is specificed.\n"
224 	    "\n"
225 	    "\tzinject -p <function name> pool\n"
226 	    "\t\tInject a panic fault at the specified function. Only \n"
227 	    "\t\tfunctions which call spa_vdev_config_exit(), or \n"
228 	    "\t\tspa_vdev_exit() will trigger a panic.\n"
229 	    "\n"
230 	    "\tzinject -d device [-e errno] [-L <nvlist|uber>] [-F]\n"
231 	    "\t    [-T <read|write|free|claim|all> pool\n"
232 	    "\t\tInject a fault into a particular device or the device's\n"
233 	    "\t\tlabel.  Label injection can either be 'nvlist' or 'uber'.\n"
234 	    "\t\t'errno' can either be 'nxio' (the default) or 'io'.\n"
235 	    "\n"
236 	    "\tzinject -d device -A <degrade|fault> pool\n"
237 	    "\t\tPerform a specific action on a particular device\n"
238 	    "\n"
239 	    "\tzinject -b objset:object:level:blkid pool\n"
240 	    "\n"
241 	    "\t\tInject an error into pool 'pool' with the numeric bookmark\n"
242 	    "\t\tspecified by the remaining tuple.  Each number is in\n"
243 	    "\t\thexidecimal, and only one block can be specified.\n"
244 	    "\n"
245 	    "\tzinject [-q] <-t type> [-e errno] [-l level] [-r range]\n"
246 	    "\t    [-a] [-m] [-u] [-f freq] <object>\n"
247 	    "\n"
248 	    "\t\tInject an error into the object specified by the '-t' option\n"
249 	    "\t\tand the object descriptor.  The 'object' parameter is\n"
250 	    "\t\tinterperted depending on the '-t' option.\n"
251 	    "\n"
252 	    "\t\t-q\tQuiet mode.  Only print out the handler number added.\n"
253 	    "\t\t-e\tInject a specific error.  Must be either 'io' or\n"
254 	    "\t\t\t'checksum'.  Default is 'io'.\n"
255 	    "\t\t-l\tInject error at a particular block level. Default is "
256 	    "0.\n"
257 	    "\t\t-m\tAutomatically remount underlying filesystem.\n"
258 	    "\t\t-r\tInject error over a particular logical range of an\n"
259 	    "\t\t\tobject.  Will be translated to the appropriate blkid\n"
260 	    "\t\t\trange according to the object's properties.\n"
261 	    "\t\t-a\tFlush the ARC cache.  Can be specified without any\n"
262 	    "\t\t\tassociated object.\n"
263 	    "\t\t-u\tUnload the associated pool.  Can be specified with only\n"
264 	    "\t\t\ta pool object.\n"
265 	    "\t\t-f\tOnly inject errors a fraction of the time.  Expressed as\n"
266 	    "\t\t\ta percentage between 1 and 100.\n"
267 	    "\n"
268 	    "\t-t data\t\tInject an error into the plain file contents of a\n"
269 	    "\t\t\tfile.  The object must be specified as a complete path\n"
270 	    "\t\t\tto a file on a ZFS filesystem.\n"
271 	    "\n"
272 	    "\t-t dnode\tInject an error into the metadnode in the block\n"
273 	    "\t\t\tcorresponding to the dnode for a file or directory.  The\n"
274 	    "\t\t\t'-r' option is incompatible with this mode.  The object\n"
275 	    "\t\t\tis specified as a complete path to a file or directory\n"
276 	    "\t\t\ton a ZFS filesystem.\n"
277 	    "\n"
278 	    "\t-t <mos>\tInject errors into the MOS for objects of the given\n"
279 	    "\t\t\ttype.  Valid types are: mos, mosdir, config, bplist,\n"
280 	    "\t\t\tspacemap, metaslab, errlog.  The only valid <object> is\n"
281 	    "\t\t\tthe poolname.\n");
282 }
283 
284 static int
285 iter_handlers(int (*func)(int, const char *, zinject_record_t *, void *),
286     void *data)
287 {
288 	zfs_cmd_t zc;
289 	int ret;
290 
291 	zc.zc_guid = 0;
292 
293 	while (ioctl(zfs_fd, ZFS_IOC_INJECT_LIST_NEXT, &zc) == 0)
294 		if ((ret = func((int)zc.zc_guid, zc.zc_name,
295 		    &zc.zc_inject_record, data)) != 0)
296 			return (ret);
297 
298 	return (0);
299 }
300 
301 static int
302 print_data_handler(int id, const char *pool, zinject_record_t *record,
303     void *data)
304 {
305 	int *count = data;
306 
307 	if (record->zi_guid != 0 || record->zi_func[0] != '\0')
308 		return (0);
309 
310 	if (*count == 0) {
311 		(void) printf("%3s  %-15s  %-6s  %-6s  %-8s  %3s  %-15s\n",
312 		    "ID", "POOL", "OBJSET", "OBJECT", "TYPE", "LVL",  "RANGE");
313 		(void) printf("---  ---------------  ------  "
314 		    "------  --------  ---  ---------------\n");
315 	}
316 
317 	*count += 1;
318 
319 	(void) printf("%3d  %-15s  %-6llu  %-6llu  %-8s  %3d  ", id, pool,
320 	    (u_longlong_t)record->zi_objset, (u_longlong_t)record->zi_object,
321 	    type_to_name(record->zi_type), record->zi_level);
322 
323 	if (record->zi_start == 0 &&
324 	    record->zi_end == -1ULL)
325 		(void) printf("all\n");
326 	else
327 		(void) printf("[%llu, %llu]\n", (u_longlong_t)record->zi_start,
328 		    (u_longlong_t)record->zi_end);
329 
330 	return (0);
331 }
332 
333 static int
334 print_device_handler(int id, const char *pool, zinject_record_t *record,
335     void *data)
336 {
337 	int *count = data;
338 
339 	if (record->zi_guid == 0 || record->zi_func[0] != '\0')
340 		return (0);
341 
342 	if (*count == 0) {
343 		(void) printf("%3s  %-15s  %s\n", "ID", "POOL", "GUID");
344 		(void) printf("---  ---------------  ----------------\n");
345 	}
346 
347 	*count += 1;
348 
349 	(void) printf("%3d  %-15s  %llx\n", id, pool,
350 	    (u_longlong_t)record->zi_guid);
351 
352 	return (0);
353 }
354 
355 static int
356 print_panic_handler(int id, const char *pool, zinject_record_t *record,
357     void *data)
358 {
359 	int *count = data;
360 
361 	if (record->zi_func[0] == '\0')
362 		return (0);
363 
364 	if (*count == 0) {
365 		(void) printf("%3s  %-15s  %s\n", "ID", "POOL", "FUNCTION");
366 		(void) printf("---  ---------------  ----------------\n");
367 	}
368 
369 	*count += 1;
370 
371 	(void) printf("%3d  %-15s  %s\n", id, pool, record->zi_func);
372 
373 	return (0);
374 }
375 
376 /*
377  * Print all registered error handlers.  Returns the number of handlers
378  * registered.
379  */
380 static int
381 print_all_handlers(void)
382 {
383 	int count = 0;
384 
385 	(void) iter_handlers(print_device_handler, &count);
386 	(void) printf("\n");
387 	count = 0;
388 	(void) iter_handlers(print_data_handler, &count);
389 	(void) printf("\n");
390 	count = 0;
391 	(void) iter_handlers(print_panic_handler, &count);
392 
393 	return (count);
394 }
395 
396 /* ARGSUSED */
397 static int
398 cancel_one_handler(int id, const char *pool, zinject_record_t *record,
399     void *data)
400 {
401 	zfs_cmd_t zc;
402 
403 	zc.zc_guid = (uint64_t)id;
404 
405 	if (ioctl(zfs_fd, ZFS_IOC_CLEAR_FAULT, &zc) != 0) {
406 		(void) fprintf(stderr, "failed to remove handler %d: %s\n",
407 		    id, strerror(errno));
408 		return (1);
409 	}
410 
411 	return (0);
412 }
413 
414 /*
415  * Remove all fault injection handlers.
416  */
417 static int
418 cancel_all_handlers(void)
419 {
420 	int ret = iter_handlers(cancel_one_handler, NULL);
421 
422 	(void) printf("removed all registered handlers\n");
423 
424 	return (ret);
425 }
426 
427 /*
428  * Remove a specific fault injection handler.
429  */
430 static int
431 cancel_handler(int id)
432 {
433 	zfs_cmd_t zc;
434 
435 	zc.zc_guid = (uint64_t)id;
436 
437 	if (ioctl(zfs_fd, ZFS_IOC_CLEAR_FAULT, &zc) != 0) {
438 		(void) fprintf(stderr, "failed to remove handler %d: %s\n",
439 		    id, strerror(errno));
440 		return (1);
441 	}
442 
443 	(void) printf("removed handler %d\n", id);
444 
445 	return (0);
446 }
447 
448 /*
449  * Register a new fault injection handler.
450  */
451 static int
452 register_handler(const char *pool, int flags, zinject_record_t *record,
453     int quiet)
454 {
455 	zfs_cmd_t zc;
456 
457 	(void) strcpy(zc.zc_name, pool);
458 	zc.zc_inject_record = *record;
459 	zc.zc_guid = flags;
460 
461 	if (ioctl(zfs_fd, ZFS_IOC_INJECT_FAULT, &zc) != 0) {
462 		(void) fprintf(stderr, "failed to add handler: %s\n",
463 		    strerror(errno));
464 		return (1);
465 	}
466 
467 	if (flags & ZINJECT_NULL)
468 		return (0);
469 
470 	if (quiet) {
471 		(void) printf("%llu\n", (u_longlong_t)zc.zc_guid);
472 	} else {
473 		(void) printf("Added handler %llu with the following "
474 		    "properties:\n", (u_longlong_t)zc.zc_guid);
475 		(void) printf("  pool: %s\n", pool);
476 		if (record->zi_guid) {
477 			(void) printf("  vdev: %llx\n",
478 			    (u_longlong_t)record->zi_guid);
479 		} else if (record->zi_func[0] != '\0') {
480 			(void) printf("  panic function: %s\n",
481 			    record->zi_func);
482 		} else {
483 			(void) printf("objset: %llu\n",
484 			    (u_longlong_t)record->zi_objset);
485 			(void) printf("object: %llu\n",
486 			    (u_longlong_t)record->zi_object);
487 			(void) printf("  type: %llu\n",
488 			    (u_longlong_t)record->zi_type);
489 			(void) printf(" level: %d\n", record->zi_level);
490 			if (record->zi_start == 0 &&
491 			    record->zi_end == -1ULL)
492 				(void) printf(" range: all\n");
493 			else
494 				(void) printf(" range: [%llu, %llu)\n",
495 				    (u_longlong_t)record->zi_start,
496 				    (u_longlong_t)record->zi_end);
497 		}
498 	}
499 
500 	return (0);
501 }
502 
503 int
504 perform_action(const char *pool, zinject_record_t *record, int cmd)
505 {
506 	zfs_cmd_t zc;
507 
508 	ASSERT(cmd == VDEV_STATE_DEGRADED || cmd == VDEV_STATE_FAULTED);
509 	(void) strlcpy(zc.zc_name, pool, sizeof (zc.zc_name));
510 	zc.zc_guid = record->zi_guid;
511 	zc.zc_cookie = cmd;
512 
513 	if (ioctl(zfs_fd, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
514 		return (0);
515 
516 	return (1);
517 }
518 
519 int
520 main(int argc, char **argv)
521 {
522 	int c;
523 	char *range = NULL;
524 	char *cancel = NULL;
525 	char *end;
526 	char *raw = NULL;
527 	char *device = NULL;
528 	int level = 0;
529 	int quiet = 0;
530 	int error = 0;
531 	int domount = 0;
532 	int io_type = ZIO_TYPES;
533 	int action = VDEV_STATE_UNKNOWN;
534 	err_type_t type = TYPE_INVAL;
535 	err_type_t label = TYPE_INVAL;
536 	zinject_record_t record = { 0 };
537 	char pool[MAXNAMELEN];
538 	char dataset[MAXNAMELEN];
539 	zfs_handle_t *zhp;
540 	int ret;
541 	int flags = 0;
542 
543 	if ((g_zfs = libzfs_init()) == NULL) {
544 		(void) fprintf(stderr, "internal error: failed to "
545 		    "initialize ZFS library\n");
546 		return (1);
547 	}
548 
549 	libzfs_print_on_error(g_zfs, B_TRUE);
550 
551 	if ((zfs_fd = open(ZFS_DEV, O_RDWR)) < 0) {
552 		(void) fprintf(stderr, "failed to open ZFS device\n");
553 		return (1);
554 	}
555 
556 	if (argc == 1) {
557 		/*
558 		 * No arguments.  Print the available handlers.  If there are no
559 		 * available handlers, direct the user to '-h' for help
560 		 * information.
561 		 */
562 		if (print_all_handlers() == 0) {
563 			(void) printf("No handlers registered.\n");
564 			(void) printf("Run 'zinject -h' for usage "
565 			    "information.\n");
566 		}
567 
568 		return (0);
569 	}
570 
571 	while ((c = getopt(argc, argv,
572 	    ":aA:b:d:f:Fqhc:t:T:l:mr:e:uL:p:")) != -1) {
573 		switch (c) {
574 		case 'a':
575 			flags |= ZINJECT_FLUSH_ARC;
576 			break;
577 		case 'A':
578 			if (strcasecmp(optarg, "degrade") == 0) {
579 				action = VDEV_STATE_DEGRADED;
580 			} else if (strcasecmp(optarg, "fault") == 0) {
581 				action = VDEV_STATE_FAULTED;
582 			} else {
583 				(void) fprintf(stderr, "invalid action '%s': "
584 				    "must be 'degrade' or 'fault'\n", optarg);
585 				usage();
586 				return (1);
587 			}
588 			break;
589 		case 'b':
590 			raw = optarg;
591 			break;
592 		case 'c':
593 			cancel = optarg;
594 			break;
595 		case 'd':
596 			device = optarg;
597 			break;
598 		case 'e':
599 			if (strcasecmp(optarg, "io") == 0) {
600 				error = EIO;
601 			} else if (strcasecmp(optarg, "checksum") == 0) {
602 				error = ECKSUM;
603 			} else if (strcasecmp(optarg, "nxio") == 0) {
604 				error = ENXIO;
605 			} else {
606 				(void) fprintf(stderr, "invalid error type "
607 				    "'%s': must be 'io', 'checksum' or "
608 				    "'nxio'\n", optarg);
609 				usage();
610 				return (1);
611 			}
612 			break;
613 		case 'f':
614 			record.zi_freq = atoi(optarg);
615 			if (record.zi_freq < 1 || record.zi_freq > 100) {
616 				(void) fprintf(stderr, "frequency range must "
617 				    "be in the range (0, 100]\n");
618 				return (1);
619 			}
620 			break;
621 		case 'F':
622 			record.zi_failfast = B_TRUE;
623 			break;
624 		case 'h':
625 			usage();
626 			return (0);
627 		case 'l':
628 			level = (int)strtol(optarg, &end, 10);
629 			if (*end != '\0') {
630 				(void) fprintf(stderr, "invalid level '%s': "
631 				    "must be an integer\n", optarg);
632 				usage();
633 				return (1);
634 			}
635 			break;
636 		case 'm':
637 			domount = 1;
638 			break;
639 		case 'p':
640 			(void) strlcpy(record.zi_func, optarg,
641 			    sizeof (record.zi_func));
642 			break;
643 		case 'q':
644 			quiet = 1;
645 			break;
646 		case 'r':
647 			range = optarg;
648 			break;
649 		case 'T':
650 			if (strcasecmp(optarg, "read") == 0) {
651 				io_type = ZIO_TYPE_READ;
652 			} else if (strcasecmp(optarg, "write") == 0) {
653 				io_type = ZIO_TYPE_WRITE;
654 			} else if (strcasecmp(optarg, "free") == 0) {
655 				io_type = ZIO_TYPE_FREE;
656 			} else if (strcasecmp(optarg, "claim") == 0) {
657 				io_type = ZIO_TYPE_CLAIM;
658 			} else if (strcasecmp(optarg, "all") == 0) {
659 				io_type = ZIO_TYPES;
660 			} else {
661 				(void) fprintf(stderr, "invalid I/O type "
662 				    "'%s': must be 'read', 'write', 'free', "
663 				    "'claim' or 'all'\n", optarg);
664 				usage();
665 				return (1);
666 			}
667 			break;
668 		case 't':
669 			if ((type = name_to_type(optarg)) == TYPE_INVAL &&
670 			    !MOS_TYPE(type)) {
671 				(void) fprintf(stderr, "invalid type '%s'\n",
672 				    optarg);
673 				usage();
674 				return (1);
675 			}
676 			break;
677 		case 'u':
678 			flags |= ZINJECT_UNLOAD_SPA;
679 			break;
680 		case 'L':
681 			if ((label = name_to_type(optarg)) == TYPE_INVAL &&
682 			    !LABEL_TYPE(type)) {
683 				(void) fprintf(stderr, "invalid label type "
684 				    "'%s'\n", optarg);
685 				usage();
686 				return (1);
687 			}
688 			break;
689 		case ':':
690 			(void) fprintf(stderr, "option -%c requires an "
691 			    "operand\n", optopt);
692 			usage();
693 			return (1);
694 		case '?':
695 			(void) fprintf(stderr, "invalid option '%c'\n",
696 			    optopt);
697 			usage();
698 			return (2);
699 		}
700 	}
701 
702 	argc -= optind;
703 	argv += optind;
704 
705 	if (cancel != NULL) {
706 		/*
707 		 * '-c' is invalid with any other options.
708 		 */
709 		if (raw != NULL || range != NULL || type != TYPE_INVAL ||
710 		    level != 0 || record.zi_func[0] != '\0') {
711 			(void) fprintf(stderr, "cancel (-c) incompatible with "
712 			    "any other options\n");
713 			usage();
714 			return (2);
715 		}
716 		if (argc != 0) {
717 			(void) fprintf(stderr, "extraneous argument to '-c'\n");
718 			usage();
719 			return (2);
720 		}
721 
722 		if (strcmp(cancel, "all") == 0) {
723 			return (cancel_all_handlers());
724 		} else {
725 			int id = (int)strtol(cancel, &end, 10);
726 			if (*end != '\0') {
727 				(void) fprintf(stderr, "invalid handle id '%s':"
728 				    " must be an integer or 'all'\n", cancel);
729 				usage();
730 				return (1);
731 			}
732 			return (cancel_handler(id));
733 		}
734 	}
735 
736 	if (device != NULL) {
737 		/*
738 		 * Device (-d) injection uses a completely different mechanism
739 		 * for doing injection, so handle it separately here.
740 		 */
741 		if (raw != NULL || range != NULL || type != TYPE_INVAL ||
742 		    level != 0 || record.zi_func[0] != '\0') {
743 			(void) fprintf(stderr, "device (-d) incompatible with "
744 			    "data error injection\n");
745 			usage();
746 			return (2);
747 		}
748 
749 		if (argc != 1) {
750 			(void) fprintf(stderr, "device (-d) injection requires "
751 			    "a single pool name\n");
752 			usage();
753 			return (2);
754 		}
755 
756 		(void) strcpy(pool, argv[0]);
757 		dataset[0] = '\0';
758 
759 		if (error == ECKSUM) {
760 			(void) fprintf(stderr, "device error type must be "
761 			    "'io' or 'nxio'\n");
762 			return (1);
763 		}
764 
765 		record.zi_iotype = io_type;
766 		if (translate_device(pool, device, label, &record) != 0)
767 			return (1);
768 		if (!error)
769 			error = ENXIO;
770 
771 		if (action != VDEV_STATE_UNKNOWN)
772 			return (perform_action(pool, &record, action));
773 
774 	} else if (raw != NULL) {
775 		if (range != NULL || type != TYPE_INVAL || level != 0 ||
776 		    record.zi_func[0] != '\0') {
777 			(void) fprintf(stderr, "raw (-b) format with "
778 			    "any other options\n");
779 			usage();
780 			return (2);
781 		}
782 
783 		if (argc != 1) {
784 			(void) fprintf(stderr, "raw (-b) format expects a "
785 			    "single pool name\n");
786 			usage();
787 			return (2);
788 		}
789 
790 		(void) strcpy(pool, argv[0]);
791 		dataset[0] = '\0';
792 
793 		if (error == ENXIO) {
794 			(void) fprintf(stderr, "data error type must be "
795 			    "'checksum' or 'io'\n");
796 			return (1);
797 		}
798 
799 		if (translate_raw(raw, &record) != 0)
800 			return (1);
801 		if (!error)
802 			error = EIO;
803 	} else if (record.zi_func[0] != '\0') {
804 		if (raw != NULL || range != NULL || type != TYPE_INVAL ||
805 		    level != 0 || device != NULL) {
806 			(void) fprintf(stderr, "panic (-p) incompatible with "
807 			    "other options\n");
808 			usage();
809 			return (2);
810 		}
811 
812 		if (argc != 1) {
813 			(void) fprintf(stderr, "panic (-p) injection requires "
814 			    "a single pool name\n");
815 			usage();
816 			return (2);
817 		}
818 
819 		(void) strcpy(pool, argv[0]);
820 		dataset[0] = '\0';
821 	} else if (type == TYPE_INVAL) {
822 		if (flags == 0) {
823 			(void) fprintf(stderr, "at least one of '-b', '-d', "
824 			    "'-t', '-a', '-p', or '-u' must be specified\n");
825 			usage();
826 			return (2);
827 		}
828 
829 		if (argc == 1 && (flags & ZINJECT_UNLOAD_SPA)) {
830 			(void) strcpy(pool, argv[0]);
831 			dataset[0] = '\0';
832 		} else if (argc != 0) {
833 			(void) fprintf(stderr, "extraneous argument for "
834 			    "'-f'\n");
835 			usage();
836 			return (2);
837 		}
838 
839 		flags |= ZINJECT_NULL;
840 	} else {
841 		if (argc != 1) {
842 			(void) fprintf(stderr, "missing object\n");
843 			usage();
844 			return (2);
845 		}
846 
847 		if (error == ENXIO) {
848 			(void) fprintf(stderr, "data error type must be "
849 			    "'checksum' or 'io'\n");
850 			return (1);
851 		}
852 
853 		if (translate_record(type, argv[0], range, level, &record, pool,
854 		    dataset) != 0)
855 			return (1);
856 		if (!error)
857 			error = EIO;
858 	}
859 
860 	/*
861 	 * If this is pool-wide metadata, unmount everything.  The ioctl() will
862 	 * unload the pool, so that we trigger spa-wide reopen of metadata next
863 	 * time we access the pool.
864 	 */
865 	if (dataset[0] != '\0' && domount) {
866 		if ((zhp = zfs_open(g_zfs, dataset, ZFS_TYPE_DATASET)) == NULL)
867 			return (1);
868 
869 		if (zfs_unmount(zhp, NULL, 0) != 0)
870 			return (1);
871 	}
872 
873 	record.zi_error = error;
874 
875 	ret = register_handler(pool, flags, &record, quiet);
876 
877 	if (dataset[0] != '\0' && domount)
878 		ret = (zfs_mount(zhp, NULL, 0) != 0);
879 
880 	libzfs_fini(g_zfs);
881 
882 	return (ret);
883 }
884