1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24 *
25 * Copyright (c) 2016, Intel Corporation.
26 * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>
27 */
28
29 /*
30 * The ZFS retire agent is responsible for managing hot spares across all pools.
31 * When we see a device fault or a device removal, we try to open the associated
32 * pool and look for any hot spares. We iterate over any available hot spares
33 * and attempt a 'zpool replace' for each one.
34 *
35 * For vdevs diagnosed as faulty, the agent is also responsible for proactively
36 * marking the vdev FAULTY (for I/O errors) or DEGRADED (for checksum errors).
37 */
38
39 #include <sys/fs/zfs.h>
40 #include <sys/fm/protocol.h>
41 #include <sys/fm/fs/zfs.h>
42 #include <libzutil.h>
43 #include <libzfs.h>
44 #include <string.h>
45 #include <libgen.h>
46
47 #include "zfs_agents.h"
48 #include "fmd_api.h"
49
50
51 typedef struct zfs_retire_repaired {
52 struct zfs_retire_repaired *zrr_next;
53 uint64_t zrr_pool;
54 uint64_t zrr_vdev;
55 } zfs_retire_repaired_t;
56
57 typedef struct zfs_retire_data {
58 libzfs_handle_t *zrd_hdl;
59 zfs_retire_repaired_t *zrd_repaired;
60 } zfs_retire_data_t;
61
62 static void
zfs_retire_clear_data(fmd_hdl_t * hdl,zfs_retire_data_t * zdp)63 zfs_retire_clear_data(fmd_hdl_t *hdl, zfs_retire_data_t *zdp)
64 {
65 zfs_retire_repaired_t *zrp;
66
67 while ((zrp = zdp->zrd_repaired) != NULL) {
68 zdp->zrd_repaired = zrp->zrr_next;
69 fmd_hdl_free(hdl, zrp, sizeof (zfs_retire_repaired_t));
70 }
71 }
72
73 /*
74 * Find a pool with a matching GUID.
75 */
76 typedef struct find_cbdata {
77 uint64_t cb_guid;
78 zpool_handle_t *cb_zhp;
79 nvlist_t *cb_vdev;
80 uint64_t cb_vdev_guid;
81 uint64_t cb_num_spares;
82 } find_cbdata_t;
83
84 static int
find_pool(zpool_handle_t * zhp,void * data)85 find_pool(zpool_handle_t *zhp, void *data)
86 {
87 find_cbdata_t *cbp = data;
88
89 if (cbp->cb_guid ==
90 zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL)) {
91 cbp->cb_zhp = zhp;
92 return (1);
93 }
94
95 zpool_close(zhp);
96 return (0);
97 }
98
99 /*
100 * Find a vdev within a tree with a matching GUID.
101 */
102 static nvlist_t *
find_vdev(libzfs_handle_t * zhdl,nvlist_t * nv,uint64_t search_guid,uint64_t * parent_guid)103 find_vdev(libzfs_handle_t *zhdl, nvlist_t *nv, uint64_t search_guid,
104 uint64_t *parent_guid)
105 {
106 uint64_t guid, saved_parent_guid;
107 nvlist_t **child;
108 uint_t c, children;
109 nvlist_t *ret = NULL;
110
111 if (parent_guid != NULL)
112 saved_parent_guid = *parent_guid;
113
114 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
115 guid == search_guid) {
116 fmd_hdl_debug(fmd_module_hdl("zfs-retire"),
117 "matched vdev %llu", guid);
118 return (nv);
119 }
120
121 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
122 &child, &children) != 0)
123 return (NULL);
124
125 for (c = 0; c < children; c++) {
126 if ((ret = find_vdev(zhdl, child[c], search_guid,
127 parent_guid)) != NULL)
128 goto out;
129 }
130
131 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
132 &child, &children) != 0)
133 return (NULL);
134
135 for (c = 0; c < children; c++) {
136 if ((ret = find_vdev(zhdl, child[c], search_guid,
137 parent_guid)) != NULL)
138 goto out;
139 }
140
141 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
142 &child, &children) != 0)
143 return (NULL);
144
145 for (c = 0; c < children; c++) {
146 if ((ret = find_vdev(zhdl, child[c], search_guid,
147 parent_guid)) != NULL)
148 goto out;
149 }
150
151 return (NULL);
152 out:
153 /* If parent_guid was set, don't reset it. */
154 if (ret != NULL && parent_guid != NULL &&
155 saved_parent_guid == *parent_guid)
156 *parent_guid = guid;
157 return (ret);
158 }
159
160 static int
remove_spares(zpool_handle_t * zhp,void * data)161 remove_spares(zpool_handle_t *zhp, void *data)
162 {
163 nvlist_t *config, *nvroot;
164 nvlist_t **spares;
165 uint_t nspares;
166 char *devname;
167 find_cbdata_t *cbp = data;
168 uint64_t spareguid = 0;
169 vdev_stat_t *vs;
170 unsigned int c;
171
172 config = zpool_get_config(zhp, NULL);
173 if (nvlist_lookup_nvlist(config,
174 ZPOOL_CONFIG_VDEV_TREE, &nvroot) != 0) {
175 zpool_close(zhp);
176 return (0);
177 }
178
179 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
180 &spares, &nspares) != 0) {
181 zpool_close(zhp);
182 return (0);
183 }
184
185 for (int i = 0; i < nspares; i++) {
186 if (nvlist_lookup_uint64(spares[i], ZPOOL_CONFIG_GUID,
187 &spareguid) == 0 && spareguid == cbp->cb_vdev_guid) {
188 devname = zpool_vdev_name(NULL, zhp, spares[i],
189 B_FALSE);
190 nvlist_lookup_uint64_array(spares[i],
191 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &c);
192 if (vs->vs_state != VDEV_STATE_REMOVED &&
193 zpool_vdev_remove_wanted(zhp, devname) == 0)
194 cbp->cb_num_spares++;
195 break;
196 }
197 }
198
199 zpool_close(zhp);
200 return (0);
201 }
202
203 /*
204 * Given a vdev guid, find and remove all spares associated with it.
205 */
206 static int
find_and_remove_spares(libzfs_handle_t * zhdl,uint64_t vdev_guid)207 find_and_remove_spares(libzfs_handle_t *zhdl, uint64_t vdev_guid)
208 {
209 find_cbdata_t cb;
210
211 cb.cb_num_spares = 0;
212 cb.cb_vdev_guid = vdev_guid;
213 zpool_iter(zhdl, remove_spares, &cb);
214
215 return (cb.cb_num_spares);
216 }
217
218 /*
219 * Given a (pool, vdev) GUID pair, find the matching pool, vdev and
220 * its top_guid.
221 */
222 static zpool_handle_t *
find_by_guid_impl(libzfs_handle_t * zhdl,uint64_t pool_guid,uint64_t vdev_guid,nvlist_t ** vdevp,uint64_t * top_guid)223 find_by_guid_impl(libzfs_handle_t *zhdl, uint64_t pool_guid, uint64_t vdev_guid,
224 nvlist_t **vdevp, uint64_t *top_guid)
225 {
226 find_cbdata_t cb;
227 zpool_handle_t *zhp;
228 nvlist_t *config, *nvroot;
229
230 /*
231 * Find the corresponding pool and make sure the vdev still exists.
232 */
233 cb.cb_guid = pool_guid;
234 if (zpool_iter(zhdl, find_pool, &cb) != 1)
235 return (NULL);
236
237 zhp = cb.cb_zhp;
238 config = zpool_get_config(zhp, NULL);
239 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
240 &nvroot) != 0) {
241 zpool_close(zhp);
242 return (NULL);
243 }
244
245 if (top_guid)
246 *top_guid = 0;
247 if (vdev_guid != 0) {
248 if ((*vdevp = find_vdev(zhdl, nvroot, vdev_guid,
249 top_guid)) == NULL) {
250 zpool_close(zhp);
251 return (NULL);
252 }
253 }
254
255 return (zhp);
256 }
257
258 /*
259 * Given a (pool, vdev) GUID pair, find the matching pool and vdev.
260 */
261 static zpool_handle_t *
find_by_guid(libzfs_handle_t * zhdl,uint64_t pool_guid,uint64_t vdev_guid,nvlist_t ** vdevp)262 find_by_guid(libzfs_handle_t *zhdl, uint64_t pool_guid, uint64_t vdev_guid,
263 nvlist_t **vdevp)
264 {
265 return (find_by_guid_impl(zhdl, pool_guid, vdev_guid, vdevp, NULL));
266 }
267
268 /*
269 * Given a (pool, vdev) GUID pair, count the number of faulted vdevs in
270 * its top vdev and return TRUE if the number of failures at i-th device
271 * index in each dRAID failure group equals to the number of failure groups,
272 * which means it's the domain failure, and the vdev is one of those faults.
273 * Otherwise, return FALSE.
274 */
275 static boolean_t
is_draid_fdomain_failure(fmd_hdl_t * hdl,libzfs_handle_t * zhdl,uint64_t pool_guid,uint64_t vdev_guid)276 is_draid_fdomain_failure(fmd_hdl_t *hdl, libzfs_handle_t *zhdl,
277 uint64_t pool_guid, uint64_t vdev_guid)
278 {
279 uint64_t guid, top_guid;
280 uint64_t children;
281 nvlist_t *nvtop, *vdev, **child;
282 vdev_stat_t *vs;
283 uint_t i, c, vdev_i = UINT_MAX, width, *nfaults_map = NULL;
284 boolean_t res = B_FALSE;
285
286 for (int try = 0; try < 4; try++) {
287 if (find_by_guid_impl(zhdl, pool_guid, vdev_guid, &vdev,
288 &top_guid) == NULL)
289 return (B_FALSE);
290
291 if (find_by_guid_impl(zhdl, pool_guid, top_guid, &nvtop,
292 NULL) == NULL)
293 return (B_FALSE);
294
295 if (nvlist_lookup_nvlist_array(nvtop, ZPOOL_CONFIG_CHILDREN,
296 &child, &width) != 0)
297 return (B_FALSE);
298
299 if (nvlist_lookup_uint64(nvtop, ZPOOL_CONFIG_DRAID_NCHILDREN,
300 &children) != 0) /* not dRAID */
301 return (B_FALSE);
302
303 if (width == children) /* dRAID without failure domains */
304 return (B_FALSE);
305
306 if (nfaults_map == NULL)
307 nfaults_map = fmd_hdl_alloc(hdl,
308 children * sizeof (*nfaults_map), FMD_SLEEP);
309 memset(nfaults_map, 0, children * sizeof (*nfaults_map));
310
311 for (c = 0; c < width; c++) {
312 nvlist_lookup_uint64_array(child[c],
313 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &i);
314
315 if (vs->vs_state != VDEV_STATE_HEALTHY)
316 nfaults_map[c % children]++;
317
318 if (vs->vs_state != VDEV_STATE_HEALTHY &&
319 nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID,
320 &guid) == 0 && guid == vdev_guid)
321 vdev_i = (c % children);
322 }
323
324 for (c = 0; c < children; c++) {
325 if (c == vdev_i &&
326 nfaults_map[c] == (width / children)) {
327 res = B_TRUE;
328 break;
329 }
330 }
331
332 if (res)
333 break;
334
335 /*
336 * No rush with starting resilver, it can be domain failure,
337 * in which case we need to wait a little to allow more devices
338 * to get into faulted state so that we could detect that
339 * it's the domain failure indeed.
340 */
341 sleep(5);
342 }
343
344 fmd_hdl_free(hdl, nfaults_map, children * sizeof (*nfaults_map));
345
346 if (res)
347 fmd_hdl_debug(hdl, "vdev %llu belongs to draid fdomain failure",
348 vdev_guid);
349
350 return (res);
351 }
352
353 /*
354 * Given a vdev, attempt to replace it with every known spare until one
355 * succeeds or we run out of devices to try.
356 * Return whether we were successful or not in replacing the device.
357 */
358 static boolean_t
replace_with_spare(fmd_hdl_t * hdl,zpool_handle_t * zhp,nvlist_t * vdev)359 replace_with_spare(fmd_hdl_t *hdl, zpool_handle_t *zhp, nvlist_t *vdev)
360 {
361 nvlist_t *config, *nvroot, *replacement;
362 nvlist_t **spares;
363 uint_t s, nspares;
364 char *dev_name;
365 zprop_source_t source;
366 int ashift;
367
368 config = zpool_get_config(zhp, NULL);
369 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
370 &nvroot) != 0)
371 return (B_FALSE);
372
373 /*
374 * Find out if there are any hot spares available in the pool.
375 */
376 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
377 &spares, &nspares) != 0)
378 return (B_FALSE);
379
380 /*
381 * lookup "ashift" pool property, we may need it for the replacement
382 */
383 ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &source);
384
385 replacement = fmd_nvl_alloc(hdl, FMD_SLEEP);
386
387 (void) nvlist_add_string(replacement, ZPOOL_CONFIG_TYPE,
388 VDEV_TYPE_ROOT);
389
390 dev_name = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);
391
392 /*
393 * Try to replace each spare, ending when we successfully
394 * replace it.
395 */
396 for (s = 0; s < nspares; s++) {
397 boolean_t rebuild = B_FALSE;
398 const char *spare_name, *type;
399
400 if (nvlist_lookup_string(spares[s], ZPOOL_CONFIG_PATH,
401 &spare_name) != 0)
402 continue;
403
404 /* prefer sequential resilvering for distributed spares */
405 if ((nvlist_lookup_string(spares[s], ZPOOL_CONFIG_TYPE,
406 &type) == 0) && strcmp(type, VDEV_TYPE_DRAID_SPARE) == 0)
407 rebuild = B_TRUE;
408
409 /* if set, add the "ashift" pool property to the spare nvlist */
410 if (source != ZPROP_SRC_DEFAULT)
411 (void) nvlist_add_uint64(spares[s],
412 ZPOOL_CONFIG_ASHIFT, ashift);
413
414 (void) nvlist_add_nvlist_array(replacement,
415 ZPOOL_CONFIG_CHILDREN, (const nvlist_t **)&spares[s], 1);
416
417 fmd_hdl_debug(hdl, "zpool_vdev_replace '%s' with spare '%s'",
418 dev_name, zfs_basename(spare_name));
419
420 if (zpool_vdev_attach(zhp, dev_name, spare_name,
421 replacement, B_TRUE, rebuild) == 0) {
422 free(dev_name);
423 nvlist_free(replacement);
424 return (B_TRUE);
425 }
426 }
427
428 free(dev_name);
429 nvlist_free(replacement);
430
431 return (B_FALSE);
432 }
433
434 /*
435 * Repair this vdev if we had diagnosed a 'fault.fs.zfs.device' and
436 * ASRU is now usable. ZFS has found the device to be present and
437 * functioning.
438 */
439 static void
zfs_vdev_repair(fmd_hdl_t * hdl,nvlist_t * nvl)440 zfs_vdev_repair(fmd_hdl_t *hdl, nvlist_t *nvl)
441 {
442 zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
443 zfs_retire_repaired_t *zrp;
444 uint64_t pool_guid, vdev_guid;
445 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
446 &pool_guid) != 0 || nvlist_lookup_uint64(nvl,
447 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
448 return;
449
450 /*
451 * Before checking the state of the ASRU, go through and see if we've
452 * already made an attempt to repair this ASRU. This list is cleared
453 * whenever we receive any kind of list event, and is designed to
454 * prevent us from generating a feedback loop when we attempt repairs
455 * against a faulted pool. The problem is that checking the unusable
456 * state of the ASRU can involve opening the pool, which can post
457 * statechange events but otherwise leave the pool in the faulted
458 * state. This list allows us to detect when a statechange event is
459 * due to our own request.
460 */
461 for (zrp = zdp->zrd_repaired; zrp != NULL; zrp = zrp->zrr_next) {
462 if (zrp->zrr_pool == pool_guid &&
463 zrp->zrr_vdev == vdev_guid)
464 return;
465 }
466
467 zrp = fmd_hdl_alloc(hdl, sizeof (zfs_retire_repaired_t), FMD_SLEEP);
468 zrp->zrr_next = zdp->zrd_repaired;
469 zrp->zrr_pool = pool_guid;
470 zrp->zrr_vdev = vdev_guid;
471 zdp->zrd_repaired = zrp;
472
473 fmd_hdl_debug(hdl, "marking repaired vdev %llu on pool %llu",
474 vdev_guid, pool_guid);
475 }
476
477 static void
zfs_retire_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)478 zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
479 const char *class)
480 {
481 (void) ep;
482 uint64_t pool_guid, vdev_guid;
483 zpool_handle_t *zhp;
484 nvlist_t *resource, *fault;
485 nvlist_t **faults;
486 uint_t f, nfaults;
487 zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
488 libzfs_handle_t *zhdl = zdp->zrd_hdl;
489 boolean_t fault_device, degrade_device;
490 boolean_t is_repair;
491 boolean_t l2arc = B_FALSE;
492 boolean_t spare = B_FALSE;
493 const char *scheme;
494 nvlist_t *vdev = NULL;
495 const char *uuid;
496 int repair_done = 0;
497 boolean_t retire;
498 boolean_t is_disk;
499 vdev_aux_t aux;
500 uint64_t state = 0;
501 vdev_stat_t *vs;
502 unsigned int c;
503
504 fmd_hdl_debug(hdl, "zfs_retire_recv: '%s'", class);
505
506 (void) nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE,
507 &state);
508
509 /*
510 * If this is a resource notifying us of device removal then simply
511 * check for an available spare and continue unless the device is a
512 * l2arc vdev, in which case we just offline it.
513 */
514 if (strcmp(class, "resource.fs.zfs.removed") == 0 ||
515 (strcmp(class, "resource.fs.zfs.statechange") == 0 &&
516 (state == VDEV_STATE_REMOVED || state == VDEV_STATE_FAULTED))) {
517 const char *devtype;
518 char *devname;
519 boolean_t skip_removal = B_FALSE;
520
521 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
522 &devtype) == 0) {
523 if (strcmp(devtype, VDEV_TYPE_SPARE) == 0)
524 spare = B_TRUE;
525 else if (strcmp(devtype, VDEV_TYPE_L2CACHE) == 0)
526 l2arc = B_TRUE;
527 }
528
529 if (nvlist_lookup_uint64(nvl,
530 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
531 return;
532
533 if (vdev_guid == 0) {
534 fmd_hdl_debug(hdl, "Got a zero GUID");
535 return;
536 }
537
538 if (spare) {
539 int nspares = find_and_remove_spares(zhdl, vdev_guid);
540 fmd_hdl_debug(hdl, "%d spares removed", nspares);
541 return;
542 }
543
544 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
545 &pool_guid) != 0)
546 return;
547
548 if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
549 &vdev)) == NULL)
550 return;
551
552 devname = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);
553
554 nvlist_lookup_uint64_array(vdev, ZPOOL_CONFIG_VDEV_STATS,
555 (uint64_t **)&vs, &c);
556
557 if (vs->vs_state == VDEV_STATE_OFFLINE)
558 return;
559
560 /*
561 * Resilvering domain failures can take a lot of computing and
562 * I/O bandwidth resources, only to be wasted when the failed
563 * domain component (for example enclosure) is replaced.
564 */
565 if (is_draid_fdomain_failure(hdl, zhdl, pool_guid, vdev_guid))
566 return;
567
568 /*
569 * If state removed is requested for already removed vdev,
570 * its a loopback event from spa_async_remove(). Just
571 * ignore it.
572 */
573 if ((vs->vs_state == VDEV_STATE_REMOVED &&
574 state == VDEV_STATE_REMOVED)) {
575 if (strcmp(class, "resource.fs.zfs.removed") == 0 &&
576 nvlist_exists(nvl, "by_kernel")) {
577 skip_removal = B_TRUE;
578 } else {
579 return;
580 }
581 }
582
583 /* Remove the vdev since device is unplugged */
584 int remove_status = 0;
585 if (!skip_removal && (l2arc ||
586 (strcmp(class, "resource.fs.zfs.removed") == 0))) {
587 remove_status = zpool_vdev_remove_wanted(zhp, devname);
588 fmd_hdl_debug(hdl, "zpool_vdev_remove_wanted '%s'"
589 ", err:%d", devname, libzfs_errno(zhdl));
590 }
591
592 /* Replace the vdev with a spare if its not a l2arc */
593 if (!l2arc && !remove_status &&
594 (!fmd_prop_get_int32(hdl, "spare_on_remove") ||
595 replace_with_spare(hdl, zhp, vdev) == B_FALSE)) {
596 /* Could not handle with spare */
597 fmd_hdl_debug(hdl, "no spare for '%s'", devname);
598 }
599
600 free(devname);
601 zpool_close(zhp);
602 return;
603 }
604
605 if (strcmp(class, FM_LIST_RESOLVED_CLASS) == 0)
606 return;
607
608 /*
609 * Note: on Linux statechange events are more than just
610 * healthy ones so we need to confirm the actual state value.
611 */
612 if (strcmp(class, "resource.fs.zfs.statechange") == 0 &&
613 state == VDEV_STATE_HEALTHY) {
614 zfs_vdev_repair(hdl, nvl);
615 return;
616 }
617 if (strcmp(class, "sysevent.fs.zfs.vdev_remove") == 0) {
618 zfs_vdev_repair(hdl, nvl);
619 return;
620 }
621
622 zfs_retire_clear_data(hdl, zdp);
623
624 if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0)
625 is_repair = B_TRUE;
626 else
627 is_repair = B_FALSE;
628
629 /*
630 * We subscribe to zfs faults as well as all repair events.
631 */
632 if (nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
633 &faults, &nfaults) != 0)
634 return;
635
636 for (f = 0; f < nfaults; f++) {
637 fault = faults[f];
638
639 fault_device = B_FALSE;
640 degrade_device = B_FALSE;
641 is_disk = B_FALSE;
642
643 if (nvlist_lookup_boolean_value(fault, FM_SUSPECT_RETIRE,
644 &retire) == 0 && retire == 0)
645 continue;
646
647 /*
648 * While we subscribe to fault.fs.zfs.*, we only take action
649 * for faults targeting a specific vdev (open failure or SERD
650 * failure). We also subscribe to fault.io.* events, so that
651 * faulty disks will be faulted in the ZFS configuration.
652 */
653 if (fmd_nvl_class_match(hdl, fault, "fault.fs.zfs.vdev.io")) {
654 fault_device = B_TRUE;
655 } else if (fmd_nvl_class_match(hdl, fault,
656 "fault.fs.zfs.vdev.checksum")) {
657 degrade_device = B_TRUE;
658 } else if (fmd_nvl_class_match(hdl, fault,
659 "fault.fs.zfs.vdev.slow_io")) {
660 degrade_device = B_TRUE;
661 } else if (fmd_nvl_class_match(hdl, fault,
662 "fault.fs.zfs.device")) {
663 fault_device = B_FALSE;
664 } else if (fmd_nvl_class_match(hdl, fault, "fault.io.*")) {
665 is_disk = B_TRUE;
666 fault_device = B_TRUE;
667 } else {
668 continue;
669 }
670
671 if (is_disk) {
672 continue;
673 } else {
674 /*
675 * This is a ZFS fault. Lookup the resource, and
676 * attempt to find the matching vdev.
677 */
678 if (nvlist_lookup_nvlist(fault, FM_FAULT_RESOURCE,
679 &resource) != 0 ||
680 nvlist_lookup_string(resource, FM_FMRI_SCHEME,
681 &scheme) != 0)
682 continue;
683
684 if (strcmp(scheme, FM_FMRI_SCHEME_ZFS) != 0)
685 continue;
686
687 if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_POOL,
688 &pool_guid) != 0)
689 continue;
690
691 if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_VDEV,
692 &vdev_guid) != 0) {
693 if (is_repair)
694 vdev_guid = 0;
695 else
696 continue;
697 }
698
699 if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
700 &vdev)) == NULL)
701 continue;
702
703 aux = VDEV_AUX_ERR_EXCEEDED;
704 }
705
706 if (vdev_guid == 0) {
707 /*
708 * For pool-level repair events, clear the entire pool.
709 */
710 fmd_hdl_debug(hdl, "zpool_clear of pool '%s'",
711 zpool_get_name(zhp));
712 (void) zpool_clear(zhp, NULL, NULL);
713 zpool_close(zhp);
714 continue;
715 }
716
717 /*
718 * If this is a repair event, then mark the vdev as repaired and
719 * continue.
720 */
721 if (is_repair) {
722 repair_done = 1;
723 fmd_hdl_debug(hdl, "zpool_clear of pool '%s' vdev %llu",
724 zpool_get_name(zhp), vdev_guid);
725 (void) zpool_vdev_clear(zhp, vdev_guid);
726 zpool_close(zhp);
727 continue;
728 }
729
730 /*
731 * Actively fault the device if needed.
732 */
733 if (fault_device)
734 (void) zpool_vdev_fault(zhp, vdev_guid, aux);
735 if (degrade_device)
736 (void) zpool_vdev_degrade(zhp, vdev_guid, aux);
737
738 if (fault_device || degrade_device)
739 fmd_hdl_debug(hdl, "zpool_vdev_%s: vdev %llu on '%s'",
740 fault_device ? "fault" : "degrade", vdev_guid,
741 zpool_get_name(zhp));
742
743 /*
744 * Attempt to substitute a hot spare.
745 */
746 (void) replace_with_spare(hdl, zhp, vdev);
747
748 zpool_close(zhp);
749 }
750
751 if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0 && repair_done &&
752 nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid) == 0)
753 fmd_case_uuresolved(hdl, uuid);
754 }
755
756 static const fmd_hdl_ops_t fmd_ops = {
757 zfs_retire_recv, /* fmdo_recv */
758 NULL, /* fmdo_timeout */
759 NULL, /* fmdo_close */
760 NULL, /* fmdo_stats */
761 NULL, /* fmdo_gc */
762 };
763
764 static const fmd_prop_t fmd_props[] = {
765 { "spare_on_remove", FMD_TYPE_BOOL, "true" },
766 { NULL, 0, NULL }
767 };
768
769 static const fmd_hdl_info_t fmd_info = {
770 "ZFS Retire Agent", "1.0", &fmd_ops, fmd_props
771 };
772
773 void
_zfs_retire_init(fmd_hdl_t * hdl)774 _zfs_retire_init(fmd_hdl_t *hdl)
775 {
776 zfs_retire_data_t *zdp;
777 libzfs_handle_t *zhdl;
778
779 if ((zhdl = libzfs_init()) == NULL)
780 return;
781
782 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
783 libzfs_fini(zhdl);
784 return;
785 }
786
787 zdp = fmd_hdl_zalloc(hdl, sizeof (zfs_retire_data_t), FMD_SLEEP);
788 zdp->zrd_hdl = zhdl;
789
790 fmd_hdl_setspecific(hdl, zdp);
791 }
792
793 void
_zfs_retire_fini(fmd_hdl_t * hdl)794 _zfs_retire_fini(fmd_hdl_t *hdl)
795 {
796 zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
797
798 if (zdp != NULL) {
799 zfs_retire_clear_data(hdl, zdp);
800 libzfs_fini(zdp->zrd_hdl);
801 fmd_hdl_free(hdl, zdp, sizeof (zfs_retire_data_t));
802 }
803 }
804