xref: /freebsd/sys/contrib/openzfs/cmd/zed/agents/zfs_retire.c (revision e2257b3168fc1697518265527e5d817eca6c43b8)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
23  *
24  * Copyright (c) 2016, Intel Corporation.
25  * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>
26  */
27 
28 /*
29  * The ZFS retire agent is responsible for managing hot spares across all pools.
30  * When we see a device fault or a device removal, we try to open the associated
31  * pool and look for any hot spares.  We iterate over any available hot spares
32  * and attempt a 'zpool replace' for each one.
33  *
34  * For vdevs diagnosed as faulty, the agent is also responsible for proactively
35  * marking the vdev FAULTY (for I/O errors) or DEGRADED (for checksum errors).
36  */
37 
38 #include <sys/fs/zfs.h>
39 #include <sys/fm/protocol.h>
40 #include <sys/fm/fs/zfs.h>
41 #include <libzutil.h>
42 #include <libzfs.h>
43 #include <string.h>
44 #include <libgen.h>
45 
46 #include "zfs_agents.h"
47 #include "fmd_api.h"
48 
49 
50 typedef struct zfs_retire_repaired {
51 	struct zfs_retire_repaired	*zrr_next;
52 	uint64_t			zrr_pool;
53 	uint64_t			zrr_vdev;
54 } zfs_retire_repaired_t;
55 
56 typedef struct zfs_retire_data {
57 	libzfs_handle_t			*zrd_hdl;
58 	zfs_retire_repaired_t		*zrd_repaired;
59 } zfs_retire_data_t;
60 
61 static void
zfs_retire_clear_data(fmd_hdl_t * hdl,zfs_retire_data_t * zdp)62 zfs_retire_clear_data(fmd_hdl_t *hdl, zfs_retire_data_t *zdp)
63 {
64 	zfs_retire_repaired_t *zrp;
65 
66 	while ((zrp = zdp->zrd_repaired) != NULL) {
67 		zdp->zrd_repaired = zrp->zrr_next;
68 		fmd_hdl_free(hdl, zrp, sizeof (zfs_retire_repaired_t));
69 	}
70 }
71 
72 /*
73  * Find a pool with a matching GUID.
74  */
75 typedef struct find_cbdata {
76 	uint64_t	cb_guid;
77 	zpool_handle_t	*cb_zhp;
78 	nvlist_t	*cb_vdev;
79 	uint64_t	cb_vdev_guid;
80 	uint64_t	cb_num_spares;
81 } find_cbdata_t;
82 
83 static int
find_pool(zpool_handle_t * zhp,void * data)84 find_pool(zpool_handle_t *zhp, void *data)
85 {
86 	find_cbdata_t *cbp = data;
87 
88 	if (cbp->cb_guid ==
89 	    zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL)) {
90 		cbp->cb_zhp = zhp;
91 		return (1);
92 	}
93 
94 	zpool_close(zhp);
95 	return (0);
96 }
97 
98 /*
99  * Find a vdev within a tree with a matching GUID.
100  */
101 static nvlist_t *
find_vdev(libzfs_handle_t * zhdl,nvlist_t * nv,uint64_t search_guid)102 find_vdev(libzfs_handle_t *zhdl, nvlist_t *nv, uint64_t search_guid)
103 {
104 	uint64_t guid;
105 	nvlist_t **child;
106 	uint_t c, children;
107 	nvlist_t *ret;
108 
109 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
110 	    guid == search_guid) {
111 		fmd_hdl_debug(fmd_module_hdl("zfs-retire"),
112 		    "matched vdev %llu", guid);
113 		return (nv);
114 	}
115 
116 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
117 	    &child, &children) != 0)
118 		return (NULL);
119 
120 	for (c = 0; c < children; c++) {
121 		if ((ret = find_vdev(zhdl, child[c], search_guid)) != NULL)
122 			return (ret);
123 	}
124 
125 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
126 	    &child, &children) != 0)
127 		return (NULL);
128 
129 	for (c = 0; c < children; c++) {
130 		if ((ret = find_vdev(zhdl, child[c], search_guid)) != NULL)
131 			return (ret);
132 	}
133 
134 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
135 	    &child, &children) != 0)
136 		return (NULL);
137 
138 	for (c = 0; c < children; c++) {
139 		if ((ret = find_vdev(zhdl, child[c], search_guid)) != NULL)
140 			return (ret);
141 	}
142 
143 	return (NULL);
144 }
145 
146 static int
remove_spares(zpool_handle_t * zhp,void * data)147 remove_spares(zpool_handle_t *zhp, void *data)
148 {
149 	nvlist_t *config, *nvroot;
150 	nvlist_t **spares;
151 	uint_t nspares;
152 	char *devname;
153 	find_cbdata_t *cbp = data;
154 	uint64_t spareguid = 0;
155 	vdev_stat_t *vs;
156 	unsigned int c;
157 
158 	config = zpool_get_config(zhp, NULL);
159 	if (nvlist_lookup_nvlist(config,
160 	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) != 0) {
161 		zpool_close(zhp);
162 		return (0);
163 	}
164 
165 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
166 	    &spares, &nspares) != 0) {
167 		zpool_close(zhp);
168 		return (0);
169 	}
170 
171 	for (int i = 0; i < nspares; i++) {
172 		if (nvlist_lookup_uint64(spares[i], ZPOOL_CONFIG_GUID,
173 		    &spareguid) == 0 && spareguid == cbp->cb_vdev_guid) {
174 			devname = zpool_vdev_name(NULL, zhp, spares[i],
175 			    B_FALSE);
176 			nvlist_lookup_uint64_array(spares[i],
177 			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &c);
178 			if (vs->vs_state != VDEV_STATE_REMOVED &&
179 			    zpool_vdev_remove_wanted(zhp, devname) == 0)
180 				cbp->cb_num_spares++;
181 			break;
182 		}
183 	}
184 
185 	zpool_close(zhp);
186 	return (0);
187 }
188 
189 /*
190  * Given a vdev guid, find and remove all spares associated with it.
191  */
192 static int
find_and_remove_spares(libzfs_handle_t * zhdl,uint64_t vdev_guid)193 find_and_remove_spares(libzfs_handle_t *zhdl, uint64_t vdev_guid)
194 {
195 	find_cbdata_t cb;
196 
197 	cb.cb_num_spares = 0;
198 	cb.cb_vdev_guid = vdev_guid;
199 	zpool_iter(zhdl, remove_spares, &cb);
200 
201 	return (cb.cb_num_spares);
202 }
203 
204 /*
205  * Given a (pool, vdev) GUID pair, find the matching pool and vdev.
206  */
207 static zpool_handle_t *
find_by_guid(libzfs_handle_t * zhdl,uint64_t pool_guid,uint64_t vdev_guid,nvlist_t ** vdevp)208 find_by_guid(libzfs_handle_t *zhdl, uint64_t pool_guid, uint64_t vdev_guid,
209     nvlist_t **vdevp)
210 {
211 	find_cbdata_t cb;
212 	zpool_handle_t *zhp;
213 	nvlist_t *config, *nvroot;
214 
215 	/*
216 	 * Find the corresponding pool and make sure the vdev still exists.
217 	 */
218 	cb.cb_guid = pool_guid;
219 	if (zpool_iter(zhdl, find_pool, &cb) != 1)
220 		return (NULL);
221 
222 	zhp = cb.cb_zhp;
223 	config = zpool_get_config(zhp, NULL);
224 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
225 	    &nvroot) != 0) {
226 		zpool_close(zhp);
227 		return (NULL);
228 	}
229 
230 	if (vdev_guid != 0) {
231 		if ((*vdevp = find_vdev(zhdl, nvroot, vdev_guid)) == NULL) {
232 			zpool_close(zhp);
233 			return (NULL);
234 		}
235 	}
236 
237 	return (zhp);
238 }
239 
240 /*
241  * Given a vdev, attempt to replace it with every known spare until one
242  * succeeds or we run out of devices to try.
243  * Return whether we were successful or not in replacing the device.
244  */
245 static boolean_t
replace_with_spare(fmd_hdl_t * hdl,zpool_handle_t * zhp,nvlist_t * vdev)246 replace_with_spare(fmd_hdl_t *hdl, zpool_handle_t *zhp, nvlist_t *vdev)
247 {
248 	nvlist_t *config, *nvroot, *replacement;
249 	nvlist_t **spares;
250 	uint_t s, nspares;
251 	char *dev_name;
252 	zprop_source_t source;
253 	int ashift;
254 
255 	config = zpool_get_config(zhp, NULL);
256 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
257 	    &nvroot) != 0)
258 		return (B_FALSE);
259 
260 	/*
261 	 * Find out if there are any hot spares available in the pool.
262 	 */
263 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
264 	    &spares, &nspares) != 0)
265 		return (B_FALSE);
266 
267 	/*
268 	 * lookup "ashift" pool property, we may need it for the replacement
269 	 */
270 	ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &source);
271 
272 	replacement = fmd_nvl_alloc(hdl, FMD_SLEEP);
273 
274 	(void) nvlist_add_string(replacement, ZPOOL_CONFIG_TYPE,
275 	    VDEV_TYPE_ROOT);
276 
277 	dev_name = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);
278 
279 	/*
280 	 * Try to replace each spare, ending when we successfully
281 	 * replace it.
282 	 */
283 	for (s = 0; s < nspares; s++) {
284 		boolean_t rebuild = B_FALSE;
285 		const char *spare_name, *type;
286 
287 		if (nvlist_lookup_string(spares[s], ZPOOL_CONFIG_PATH,
288 		    &spare_name) != 0)
289 			continue;
290 
291 		/* prefer sequential resilvering for distributed spares */
292 		if ((nvlist_lookup_string(spares[s], ZPOOL_CONFIG_TYPE,
293 		    &type) == 0) && strcmp(type, VDEV_TYPE_DRAID_SPARE) == 0)
294 			rebuild = B_TRUE;
295 
296 		/* if set, add the "ashift" pool property to the spare nvlist */
297 		if (source != ZPROP_SRC_DEFAULT)
298 			(void) nvlist_add_uint64(spares[s],
299 			    ZPOOL_CONFIG_ASHIFT, ashift);
300 
301 		(void) nvlist_add_nvlist_array(replacement,
302 		    ZPOOL_CONFIG_CHILDREN, (const nvlist_t **)&spares[s], 1);
303 
304 		fmd_hdl_debug(hdl, "zpool_vdev_replace '%s' with spare '%s'",
305 		    dev_name, zfs_basename(spare_name));
306 
307 		if (zpool_vdev_attach(zhp, dev_name, spare_name,
308 		    replacement, B_TRUE, rebuild) == 0) {
309 			free(dev_name);
310 			nvlist_free(replacement);
311 			return (B_TRUE);
312 		}
313 	}
314 
315 	free(dev_name);
316 	nvlist_free(replacement);
317 
318 	return (B_FALSE);
319 }
320 
321 /*
322  * Repair this vdev if we had diagnosed a 'fault.fs.zfs.device' and
323  * ASRU is now usable.  ZFS has found the device to be present and
324  * functioning.
325  */
326 static void
zfs_vdev_repair(fmd_hdl_t * hdl,nvlist_t * nvl)327 zfs_vdev_repair(fmd_hdl_t *hdl, nvlist_t *nvl)
328 {
329 	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
330 	zfs_retire_repaired_t *zrp;
331 	uint64_t pool_guid, vdev_guid;
332 	if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
333 	    &pool_guid) != 0 || nvlist_lookup_uint64(nvl,
334 	    FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
335 		return;
336 
337 	/*
338 	 * Before checking the state of the ASRU, go through and see if we've
339 	 * already made an attempt to repair this ASRU.  This list is cleared
340 	 * whenever we receive any kind of list event, and is designed to
341 	 * prevent us from generating a feedback loop when we attempt repairs
342 	 * against a faulted pool.  The problem is that checking the unusable
343 	 * state of the ASRU can involve opening the pool, which can post
344 	 * statechange events but otherwise leave the pool in the faulted
345 	 * state.  This list allows us to detect when a statechange event is
346 	 * due to our own request.
347 	 */
348 	for (zrp = zdp->zrd_repaired; zrp != NULL; zrp = zrp->zrr_next) {
349 		if (zrp->zrr_pool == pool_guid &&
350 		    zrp->zrr_vdev == vdev_guid)
351 			return;
352 	}
353 
354 	zrp = fmd_hdl_alloc(hdl, sizeof (zfs_retire_repaired_t), FMD_SLEEP);
355 	zrp->zrr_next = zdp->zrd_repaired;
356 	zrp->zrr_pool = pool_guid;
357 	zrp->zrr_vdev = vdev_guid;
358 	zdp->zrd_repaired = zrp;
359 
360 	fmd_hdl_debug(hdl, "marking repaired vdev %llu on pool %llu",
361 	    vdev_guid, pool_guid);
362 }
363 
364 static void
zfs_retire_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)365 zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
366     const char *class)
367 {
368 	(void) ep;
369 	uint64_t pool_guid, vdev_guid;
370 	zpool_handle_t *zhp;
371 	nvlist_t *resource, *fault;
372 	nvlist_t **faults;
373 	uint_t f, nfaults;
374 	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
375 	libzfs_handle_t *zhdl = zdp->zrd_hdl;
376 	boolean_t fault_device, degrade_device;
377 	boolean_t is_repair;
378 	boolean_t l2arc = B_FALSE;
379 	boolean_t spare = B_FALSE;
380 	const char *scheme;
381 	nvlist_t *vdev = NULL;
382 	const char *uuid;
383 	int repair_done = 0;
384 	boolean_t retire;
385 	boolean_t is_disk;
386 	vdev_aux_t aux;
387 	uint64_t state = 0;
388 	vdev_stat_t *vs;
389 	unsigned int c;
390 
391 	fmd_hdl_debug(hdl, "zfs_retire_recv: '%s'", class);
392 
393 	(void) nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE,
394 	    &state);
395 
396 	/*
397 	 * If this is a resource notifying us of device removal then simply
398 	 * check for an available spare and continue unless the device is a
399 	 * l2arc vdev, in which case we just offline it.
400 	 */
401 	if (strcmp(class, "resource.fs.zfs.removed") == 0 ||
402 	    (strcmp(class, "resource.fs.zfs.statechange") == 0 &&
403 	    (state == VDEV_STATE_REMOVED || state == VDEV_STATE_FAULTED))) {
404 		const char *devtype;
405 		char *devname;
406 
407 		if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
408 		    &devtype) == 0) {
409 			if (strcmp(devtype, VDEV_TYPE_SPARE) == 0)
410 				spare = B_TRUE;
411 			else if (strcmp(devtype, VDEV_TYPE_L2CACHE) == 0)
412 				l2arc = B_TRUE;
413 		}
414 
415 		if (nvlist_lookup_uint64(nvl,
416 		    FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
417 			return;
418 
419 		if (vdev_guid == 0) {
420 			fmd_hdl_debug(hdl, "Got a zero GUID");
421 			return;
422 		}
423 
424 		if (spare) {
425 			int nspares = find_and_remove_spares(zhdl, vdev_guid);
426 			fmd_hdl_debug(hdl, "%d spares removed", nspares);
427 			return;
428 		}
429 
430 		if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
431 		    &pool_guid) != 0)
432 			return;
433 
434 		if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
435 		    &vdev)) == NULL)
436 			return;
437 
438 		devname = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);
439 
440 		nvlist_lookup_uint64_array(vdev, ZPOOL_CONFIG_VDEV_STATS,
441 		    (uint64_t **)&vs, &c);
442 
443 		/*
444 		 * If state removed is requested for already removed vdev,
445 		 * its a loopback event from spa_async_remove(). Just
446 		 * ignore it.
447 		 */
448 		if (vs->vs_state == VDEV_STATE_REMOVED &&
449 		    state == VDEV_STATE_REMOVED)
450 			return;
451 
452 		/* Remove the vdev since device is unplugged */
453 		int remove_status = 0;
454 		if (l2arc || (strcmp(class, "resource.fs.zfs.removed") == 0)) {
455 			remove_status = zpool_vdev_remove_wanted(zhp, devname);
456 			fmd_hdl_debug(hdl, "zpool_vdev_remove_wanted '%s'"
457 			    ", err:%d", devname, libzfs_errno(zhdl));
458 		}
459 
460 		/* Replace the vdev with a spare if its not a l2arc */
461 		if (!l2arc && !remove_status &&
462 		    (!fmd_prop_get_int32(hdl, "spare_on_remove") ||
463 		    replace_with_spare(hdl, zhp, vdev) == B_FALSE)) {
464 			/* Could not handle with spare */
465 			fmd_hdl_debug(hdl, "no spare for '%s'", devname);
466 		}
467 
468 		free(devname);
469 		zpool_close(zhp);
470 		return;
471 	}
472 
473 	if (strcmp(class, FM_LIST_RESOLVED_CLASS) == 0)
474 		return;
475 
476 	/*
477 	 * Note: on Linux statechange events are more than just
478 	 * healthy ones so we need to confirm the actual state value.
479 	 */
480 	if (strcmp(class, "resource.fs.zfs.statechange") == 0 &&
481 	    state == VDEV_STATE_HEALTHY) {
482 		zfs_vdev_repair(hdl, nvl);
483 		return;
484 	}
485 	if (strcmp(class, "sysevent.fs.zfs.vdev_remove") == 0) {
486 		zfs_vdev_repair(hdl, nvl);
487 		return;
488 	}
489 
490 	zfs_retire_clear_data(hdl, zdp);
491 
492 	if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0)
493 		is_repair = B_TRUE;
494 	else
495 		is_repair = B_FALSE;
496 
497 	/*
498 	 * We subscribe to zfs faults as well as all repair events.
499 	 */
500 	if (nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
501 	    &faults, &nfaults) != 0)
502 		return;
503 
504 	for (f = 0; f < nfaults; f++) {
505 		fault = faults[f];
506 
507 		fault_device = B_FALSE;
508 		degrade_device = B_FALSE;
509 		is_disk = B_FALSE;
510 
511 		if (nvlist_lookup_boolean_value(fault, FM_SUSPECT_RETIRE,
512 		    &retire) == 0 && retire == 0)
513 			continue;
514 
515 		/*
516 		 * While we subscribe to fault.fs.zfs.*, we only take action
517 		 * for faults targeting a specific vdev (open failure or SERD
518 		 * failure).  We also subscribe to fault.io.* events, so that
519 		 * faulty disks will be faulted in the ZFS configuration.
520 		 */
521 		if (fmd_nvl_class_match(hdl, fault, "fault.fs.zfs.vdev.io")) {
522 			fault_device = B_TRUE;
523 		} else if (fmd_nvl_class_match(hdl, fault,
524 		    "fault.fs.zfs.vdev.checksum")) {
525 			degrade_device = B_TRUE;
526 		} else if (fmd_nvl_class_match(hdl, fault,
527 		    "fault.fs.zfs.vdev.slow_io")) {
528 			degrade_device = B_TRUE;
529 		} else if (fmd_nvl_class_match(hdl, fault,
530 		    "fault.fs.zfs.device")) {
531 			fault_device = B_FALSE;
532 		} else if (fmd_nvl_class_match(hdl, fault, "fault.io.*")) {
533 			is_disk = B_TRUE;
534 			fault_device = B_TRUE;
535 		} else {
536 			continue;
537 		}
538 
539 		if (is_disk) {
540 			continue;
541 		} else {
542 			/*
543 			 * This is a ZFS fault.  Lookup the resource, and
544 			 * attempt to find the matching vdev.
545 			 */
546 			if (nvlist_lookup_nvlist(fault, FM_FAULT_RESOURCE,
547 			    &resource) != 0 ||
548 			    nvlist_lookup_string(resource, FM_FMRI_SCHEME,
549 			    &scheme) != 0)
550 				continue;
551 
552 			if (strcmp(scheme, FM_FMRI_SCHEME_ZFS) != 0)
553 				continue;
554 
555 			if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_POOL,
556 			    &pool_guid) != 0)
557 				continue;
558 
559 			if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_VDEV,
560 			    &vdev_guid) != 0) {
561 				if (is_repair)
562 					vdev_guid = 0;
563 				else
564 					continue;
565 			}
566 
567 			if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
568 			    &vdev)) == NULL)
569 				continue;
570 
571 			aux = VDEV_AUX_ERR_EXCEEDED;
572 		}
573 
574 		if (vdev_guid == 0) {
575 			/*
576 			 * For pool-level repair events, clear the entire pool.
577 			 */
578 			fmd_hdl_debug(hdl, "zpool_clear of pool '%s'",
579 			    zpool_get_name(zhp));
580 			(void) zpool_clear(zhp, NULL, NULL);
581 			zpool_close(zhp);
582 			continue;
583 		}
584 
585 		/*
586 		 * If this is a repair event, then mark the vdev as repaired and
587 		 * continue.
588 		 */
589 		if (is_repair) {
590 			repair_done = 1;
591 			fmd_hdl_debug(hdl, "zpool_clear of pool '%s' vdev %llu",
592 			    zpool_get_name(zhp), vdev_guid);
593 			(void) zpool_vdev_clear(zhp, vdev_guid);
594 			zpool_close(zhp);
595 			continue;
596 		}
597 
598 		/*
599 		 * Actively fault the device if needed.
600 		 */
601 		if (fault_device)
602 			(void) zpool_vdev_fault(zhp, vdev_guid, aux);
603 		if (degrade_device)
604 			(void) zpool_vdev_degrade(zhp, vdev_guid, aux);
605 
606 		if (fault_device || degrade_device)
607 			fmd_hdl_debug(hdl, "zpool_vdev_%s: vdev %llu on '%s'",
608 			    fault_device ? "fault" : "degrade", vdev_guid,
609 			    zpool_get_name(zhp));
610 
611 		/*
612 		 * Attempt to substitute a hot spare.
613 		 */
614 		(void) replace_with_spare(hdl, zhp, vdev);
615 
616 		zpool_close(zhp);
617 	}
618 
619 	if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0 && repair_done &&
620 	    nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid) == 0)
621 		fmd_case_uuresolved(hdl, uuid);
622 }
623 
624 static const fmd_hdl_ops_t fmd_ops = {
625 	zfs_retire_recv,	/* fmdo_recv */
626 	NULL,			/* fmdo_timeout */
627 	NULL,			/* fmdo_close */
628 	NULL,			/* fmdo_stats */
629 	NULL,			/* fmdo_gc */
630 };
631 
632 static const fmd_prop_t fmd_props[] = {
633 	{ "spare_on_remove", FMD_TYPE_BOOL, "true" },
634 	{ NULL, 0, NULL }
635 };
636 
637 static const fmd_hdl_info_t fmd_info = {
638 	"ZFS Retire Agent", "1.0", &fmd_ops, fmd_props
639 };
640 
641 void
_zfs_retire_init(fmd_hdl_t * hdl)642 _zfs_retire_init(fmd_hdl_t *hdl)
643 {
644 	zfs_retire_data_t *zdp;
645 	libzfs_handle_t *zhdl;
646 
647 	if ((zhdl = libzfs_init()) == NULL)
648 		return;
649 
650 	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
651 		libzfs_fini(zhdl);
652 		return;
653 	}
654 
655 	zdp = fmd_hdl_zalloc(hdl, sizeof (zfs_retire_data_t), FMD_SLEEP);
656 	zdp->zrd_hdl = zhdl;
657 
658 	fmd_hdl_setspecific(hdl, zdp);
659 }
660 
661 void
_zfs_retire_fini(fmd_hdl_t * hdl)662 _zfs_retire_fini(fmd_hdl_t *hdl)
663 {
664 	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
665 
666 	if (zdp != NULL) {
667 		zfs_retire_clear_data(hdl, zdp);
668 		libzfs_fini(zdp->zrd_hdl);
669 		fmd_hdl_free(hdl, zdp, sizeof (zfs_retire_data_t));
670 	}
671 }
672