1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
26 * Copyright (c) 2016, Intel Corporation.
27 * Copyright (c) 2023, Klara Inc.
28 */
29
30 #include <stddef.h>
31 #include <string.h>
32 #include <libuutil.h>
33 #include <libzfs.h>
34 #include <sys/types.h>
35 #include <sys/time.h>
36 #include <sys/fs/zfs.h>
37 #include <sys/fm/protocol.h>
38 #include <sys/fm/fs/zfs.h>
39 #include <sys/zio.h>
40
41 #include "zfs_agents.h"
42 #include "fmd_api.h"
43
44 /*
45 * Default values for the serd engine when processing checksum or io errors. The
46 * semantics are N <events> in T <seconds>.
47 */
48 #define DEFAULT_CHECKSUM_N 10 /* events */
49 #define DEFAULT_CHECKSUM_T 600 /* seconds */
50 #define DEFAULT_IO_N 10 /* events */
51 #define DEFAULT_IO_T 600 /* seconds */
52 #define DEFAULT_SLOW_IO_N 10 /* events */
53 #define DEFAULT_SLOW_IO_T 30 /* seconds */
54
55 #define CASE_GC_TIMEOUT_SECS 43200 /* 12 hours */
56
57 /*
58 * Our serd engines are named in the following format:
59 * 'zfs_<pool_guid>_<vdev_guid>_{checksum,io,slow_io}'
60 * This #define reserves enough space for two 64-bit hex values plus the
61 * length of the longest string.
62 */
63 #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))
64
65 /*
66 * On-disk case structure. This must maintain backwards compatibility with
67 * previous versions of the DE. By default, any members appended to the end
68 * will be filled with zeros if they don't exist in a previous version.
69 */
70 typedef struct zfs_case_data {
71 uint64_t zc_version;
72 uint64_t zc_ena;
73 uint64_t zc_pool_guid;
74 uint64_t zc_vdev_guid;
75 uint64_t zc_parent_guid;
76 int zc_pool_state;
77 char zc_serd_checksum[MAX_SERDLEN];
78 char zc_serd_io[MAX_SERDLEN];
79 char zc_serd_slow_io[MAX_SERDLEN];
80 int zc_has_remove_timer;
81 } zfs_case_data_t;
82
83 /*
84 * Time-of-day
85 */
86 typedef struct er_timeval {
87 uint64_t ertv_sec;
88 uint64_t ertv_nsec;
89 } er_timeval_t;
90
91 /*
92 * In-core case structure.
93 */
94 typedef struct zfs_case {
95 boolean_t zc_present;
96 uint32_t zc_version;
97 zfs_case_data_t zc_data;
98 fmd_case_t *zc_case;
99 uu_list_node_t zc_node;
100 id_t zc_remove_timer;
101 char *zc_fru;
102 er_timeval_t zc_when;
103 } zfs_case_t;
104
105 #define CASE_DATA "data"
106 #define CASE_FRU "fru"
107 #define CASE_DATA_VERSION_INITIAL 1
108 #define CASE_DATA_VERSION_SERD 2
109
110 typedef struct zfs_de_stats {
111 fmd_stat_t old_drops;
112 fmd_stat_t dev_drops;
113 fmd_stat_t vdev_drops;
114 fmd_stat_t import_drops;
115 fmd_stat_t resource_drops;
116 } zfs_de_stats_t;
117
118 zfs_de_stats_t zfs_stats = {
119 { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
120 { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
121 { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
122 { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
123 { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
124 };
125
126 /* wait 15 seconds after a removal */
127 static hrtime_t zfs_remove_timeout = SEC2NSEC(15);
128
129 uu_list_pool_t *zfs_case_pool;
130 uu_list_t *zfs_cases;
131
132 #define ZFS_MAKE_RSRC(type) \
133 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
134 #define ZFS_MAKE_EREPORT(type) \
135 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
136
137 static void zfs_purge_cases(fmd_hdl_t *hdl);
138
139 /*
140 * Write out the persistent representation of an active case.
141 */
142 static void
zfs_case_serialize(zfs_case_t * zcp)143 zfs_case_serialize(zfs_case_t *zcp)
144 {
145 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
146 }
147
148 /*
149 * Read back the persistent representation of an active case.
150 */
151 static zfs_case_t *
zfs_case_unserialize(fmd_hdl_t * hdl,fmd_case_t * cp)152 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
153 {
154 zfs_case_t *zcp;
155
156 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
157 zcp->zc_case = cp;
158
159 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
160 sizeof (zcp->zc_data));
161
162 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
163 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
164 return (NULL);
165 }
166
167 /*
168 * fmd_buf_read() will have already zeroed out the remainder of the
169 * buffer, so we don't have to do anything special if the version
170 * doesn't include the SERD engine name.
171 */
172
173 if (zcp->zc_data.zc_has_remove_timer)
174 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
175 NULL, zfs_remove_timeout);
176
177 uu_list_node_init(zcp, &zcp->zc_node, zfs_case_pool);
178 (void) uu_list_insert_before(zfs_cases, NULL, zcp);
179
180 fmd_case_setspecific(hdl, cp, zcp);
181
182 return (zcp);
183 }
184
185 /*
186 * Return count of other unique SERD cases under same vdev parent
187 */
188 static uint_t
zfs_other_serd_cases(fmd_hdl_t * hdl,const zfs_case_data_t * zfs_case)189 zfs_other_serd_cases(fmd_hdl_t *hdl, const zfs_case_data_t *zfs_case)
190 {
191 zfs_case_t *zcp;
192 uint_t cases = 0;
193 static hrtime_t next_check = 0;
194
195 /*
196 * Note that plumbing in some external GC would require adding locking,
197 * since most of this module code is not thread safe and assumes there
198 * is only one thread running against the module. So we perform GC here
199 * inline periodically so that future delay induced faults will be
200 * possible once the issue causing multiple vdev delays is resolved.
201 */
202 if (gethrestime_sec() > next_check) {
203 /* Periodically purge old SERD entries and stale cases */
204 fmd_serd_gc(hdl);
205 zfs_purge_cases(hdl);
206 next_check = gethrestime_sec() + CASE_GC_TIMEOUT_SECS;
207 }
208
209 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
210 zcp = uu_list_next(zfs_cases, zcp)) {
211 zfs_case_data_t *zcd = &zcp->zc_data;
212
213 /*
214 * must be same pool and parent vdev but different leaf vdev
215 */
216 if (zcd->zc_pool_guid != zfs_case->zc_pool_guid ||
217 zcd->zc_parent_guid != zfs_case->zc_parent_guid ||
218 zcd->zc_vdev_guid == zfs_case->zc_vdev_guid) {
219 continue;
220 }
221
222 /*
223 * Check if there is another active serd case besides zfs_case
224 *
225 * Only one serd engine will be assigned to the case
226 */
227 if (zcd->zc_serd_checksum[0] == zfs_case->zc_serd_checksum[0] &&
228 fmd_serd_active(hdl, zcd->zc_serd_checksum)) {
229 cases++;
230 }
231 if (zcd->zc_serd_io[0] == zfs_case->zc_serd_io[0] &&
232 fmd_serd_active(hdl, zcd->zc_serd_io)) {
233 cases++;
234 }
235 if (zcd->zc_serd_slow_io[0] == zfs_case->zc_serd_slow_io[0] &&
236 fmd_serd_active(hdl, zcd->zc_serd_slow_io)) {
237 cases++;
238 }
239 }
240 return (cases);
241 }
242
243 /*
244 * Iterate over any active cases. If any cases are associated with a pool or
245 * vdev which is no longer present on the system, close the associated case.
246 */
247 static void
zfs_mark_vdev(uint64_t pool_guid,nvlist_t * vd,er_timeval_t * loaded)248 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
249 {
250 uint64_t vdev_guid = 0;
251 uint_t c, children;
252 nvlist_t **child;
253 zfs_case_t *zcp;
254
255 (void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
256
257 /*
258 * Mark any cases associated with this (pool, vdev) pair.
259 */
260 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
261 zcp = uu_list_next(zfs_cases, zcp)) {
262 if (zcp->zc_data.zc_pool_guid == pool_guid &&
263 zcp->zc_data.zc_vdev_guid == vdev_guid) {
264 zcp->zc_present = B_TRUE;
265 zcp->zc_when = *loaded;
266 }
267 }
268
269 /*
270 * Iterate over all children.
271 */
272 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
273 &children) == 0) {
274 for (c = 0; c < children; c++)
275 zfs_mark_vdev(pool_guid, child[c], loaded);
276 }
277
278 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
279 &children) == 0) {
280 for (c = 0; c < children; c++)
281 zfs_mark_vdev(pool_guid, child[c], loaded);
282 }
283
284 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
285 &children) == 0) {
286 for (c = 0; c < children; c++)
287 zfs_mark_vdev(pool_guid, child[c], loaded);
288 }
289 }
290
291 static int
zfs_mark_pool(zpool_handle_t * zhp,void * unused)292 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
293 {
294 (void) unused;
295 zfs_case_t *zcp;
296 uint64_t pool_guid;
297 uint64_t *tod;
298 er_timeval_t loaded = { 0 };
299 nvlist_t *config, *vd;
300 uint_t nelem = 0;
301 int ret;
302
303 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
304 /*
305 * Mark any cases associated with just this pool.
306 */
307 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
308 zcp = uu_list_next(zfs_cases, zcp)) {
309 if (zcp->zc_data.zc_pool_guid == pool_guid &&
310 zcp->zc_data.zc_vdev_guid == 0)
311 zcp->zc_present = B_TRUE;
312 }
313
314 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
315 zpool_close(zhp);
316 return (-1);
317 }
318
319 (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
320 &tod, &nelem);
321 if (nelem == 2) {
322 loaded.ertv_sec = tod[0];
323 loaded.ertv_nsec = tod[1];
324 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
325 zcp = uu_list_next(zfs_cases, zcp)) {
326 if (zcp->zc_data.zc_pool_guid == pool_guid &&
327 zcp->zc_data.zc_vdev_guid == 0) {
328 zcp->zc_when = loaded;
329 }
330 }
331 }
332
333 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
334 if (ret) {
335 zpool_close(zhp);
336 return (-1);
337 }
338
339 zfs_mark_vdev(pool_guid, vd, &loaded);
340
341 zpool_close(zhp);
342
343 return (0);
344 }
345
346 struct load_time_arg {
347 uint64_t lt_guid;
348 er_timeval_t *lt_time;
349 boolean_t lt_found;
350 };
351
352 static int
zpool_find_load_time(zpool_handle_t * zhp,void * arg)353 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
354 {
355 struct load_time_arg *lta = arg;
356 uint64_t pool_guid;
357 uint64_t *tod;
358 nvlist_t *config;
359 uint_t nelem;
360
361 if (lta->lt_found) {
362 zpool_close(zhp);
363 return (0);
364 }
365
366 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
367 if (pool_guid != lta->lt_guid) {
368 zpool_close(zhp);
369 return (0);
370 }
371
372 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
373 zpool_close(zhp);
374 return (-1);
375 }
376
377 if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
378 &tod, &nelem) == 0 && nelem == 2) {
379 lta->lt_found = B_TRUE;
380 lta->lt_time->ertv_sec = tod[0];
381 lta->lt_time->ertv_nsec = tod[1];
382 }
383
384 zpool_close(zhp);
385
386 return (0);
387 }
388
389 static void
zfs_purge_cases(fmd_hdl_t * hdl)390 zfs_purge_cases(fmd_hdl_t *hdl)
391 {
392 zfs_case_t *zcp;
393 uu_list_walk_t *walk;
394 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
395
396 /*
397 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No
398 * matter what we do, we're going to have to stomach an O(vdevs * cases)
399 * algorithm. In reality, both quantities are likely so small that
400 * neither will matter. Given that iterating over pools is more
401 * expensive than iterating over the in-memory case list, we opt for a
402 * 'present' flag in each case that starts off cleared. We then iterate
403 * over all pools, marking those that are still present, and removing
404 * those that aren't found.
405 *
406 * Note that we could also construct an FMRI and rely on
407 * fmd_nvl_fmri_present(), but this would end up doing the same search.
408 */
409
410 /*
411 * Mark the cases as not present.
412 */
413 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
414 zcp = uu_list_next(zfs_cases, zcp))
415 zcp->zc_present = B_FALSE;
416
417 /*
418 * Iterate over all pools and mark the pools and vdevs found. If this
419 * fails (most probably because we're out of memory), then don't close
420 * any of the cases and we cannot be sure they are accurate.
421 */
422 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
423 return;
424
425 /*
426 * Remove those cases which were not found.
427 */
428 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
429 while ((zcp = uu_list_walk_next(walk)) != NULL) {
430 if (!zcp->zc_present)
431 fmd_case_close(hdl, zcp->zc_case);
432 }
433 uu_list_walk_end(walk);
434 }
435
436 /*
437 * Construct the name of a serd engine given the pool/vdev GUID and type (io or
438 * checksum).
439 */
440 static void
zfs_serd_name(char * buf,uint64_t pool_guid,uint64_t vdev_guid,const char * type)441 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
442 const char *type)
443 {
444 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s",
445 (long long unsigned int)pool_guid,
446 (long long unsigned int)vdev_guid, type);
447 }
448
449 static void
zfs_case_retire(fmd_hdl_t * hdl,zfs_case_t * zcp)450 zfs_case_retire(fmd_hdl_t *hdl, zfs_case_t *zcp)
451 {
452 fmd_hdl_debug(hdl, "retiring case");
453
454 fmd_case_close(hdl, zcp->zc_case);
455 }
456
457 /*
458 * Solve a given ZFS case. This first checks to make sure the diagnosis is
459 * still valid, as well as cleaning up any pending timer associated with the
460 * case.
461 */
462 static void
zfs_case_solve(fmd_hdl_t * hdl,zfs_case_t * zcp,const char * faultname)463 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname)
464 {
465 nvlist_t *detector, *fault;
466 boolean_t serialize;
467 nvlist_t *fru = NULL;
468 fmd_hdl_debug(hdl, "solving fault '%s'", faultname);
469
470 /*
471 * Construct the detector from the case data. The detector is in the
472 * ZFS scheme, and is either the pool or the vdev, depending on whether
473 * this is a vdev or pool fault.
474 */
475 detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
476
477 (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
478 (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
479 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
480 zcp->zc_data.zc_pool_guid);
481 if (zcp->zc_data.zc_vdev_guid != 0) {
482 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
483 zcp->zc_data.zc_vdev_guid);
484 }
485
486 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
487 fru, detector);
488 fmd_case_add_suspect(hdl, zcp->zc_case, fault);
489
490 nvlist_free(fru);
491
492 fmd_case_solve(hdl, zcp->zc_case);
493
494 serialize = B_FALSE;
495 if (zcp->zc_data.zc_has_remove_timer) {
496 fmd_timer_remove(hdl, zcp->zc_remove_timer);
497 zcp->zc_data.zc_has_remove_timer = 0;
498 serialize = B_TRUE;
499 }
500 if (serialize)
501 zfs_case_serialize(zcp);
502
503 nvlist_free(detector);
504 }
505
506 static boolean_t
timeval_earlier(er_timeval_t * a,er_timeval_t * b)507 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
508 {
509 return (a->ertv_sec < b->ertv_sec ||
510 (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
511 }
512
513 static void
zfs_ereport_when(fmd_hdl_t * hdl,nvlist_t * nvl,er_timeval_t * when)514 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
515 {
516 (void) hdl;
517 int64_t *tod;
518 uint_t nelem;
519
520 if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod,
521 &nelem) == 0 && nelem == 2) {
522 when->ertv_sec = tod[0];
523 when->ertv_nsec = tod[1];
524 } else {
525 when->ertv_sec = when->ertv_nsec = UINT64_MAX;
526 }
527 }
528
529 /*
530 * Record the specified event in the SERD engine and return a
531 * boolean value indicating whether or not the engine fired as
532 * the result of inserting this event.
533 *
534 * When the pool has similar active cases on other vdevs, then
535 * the fired state is disregarded and the case is retired.
536 */
537 static int
zfs_fm_serd_record(fmd_hdl_t * hdl,const char * name,fmd_event_t * ep,zfs_case_t * zcp,const char * err_type)538 zfs_fm_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep,
539 zfs_case_t *zcp, const char *err_type)
540 {
541 int fired = fmd_serd_record(hdl, name, ep);
542 int peers = 0;
543
544 if (fired && (peers = zfs_other_serd_cases(hdl, &zcp->zc_data)) > 0) {
545 fmd_hdl_debug(hdl, "pool %llu is tracking %d other %s cases "
546 "-- skip faulting the vdev %llu",
547 (u_longlong_t)zcp->zc_data.zc_pool_guid,
548 peers, err_type,
549 (u_longlong_t)zcp->zc_data.zc_vdev_guid);
550 zfs_case_retire(hdl, zcp);
551 fired = 0;
552 }
553
554 return (fired);
555 }
556
557 /*
558 * Main fmd entry point.
559 */
560 static void
zfs_fm_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)561 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
562 {
563 zfs_case_t *zcp, *dcp;
564 int32_t pool_state;
565 uint64_t ena, pool_guid, vdev_guid, parent_guid;
566 uint64_t checksum_n, checksum_t;
567 uint64_t io_n, io_t;
568 er_timeval_t pool_load;
569 er_timeval_t er_when;
570 nvlist_t *detector;
571 boolean_t pool_found = B_FALSE;
572 boolean_t isresource;
573 const char *type;
574
575 /*
576 * We subscribe to notifications for vdev or pool removal. In these
577 * cases, there may be cases that no longer apply. Purge any cases
578 * that no longer apply.
579 */
580 if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) {
581 fmd_hdl_debug(hdl, "purging orphaned cases from %s",
582 strrchr(class, '.') + 1);
583 zfs_purge_cases(hdl);
584 zfs_stats.resource_drops.fmds_value.ui64++;
585 return;
586 }
587
588 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
589
590 if (isresource) {
591 /*
592 * For resources, we don't have a normal payload.
593 */
594 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
595 &vdev_guid) != 0)
596 pool_state = SPA_LOAD_OPEN;
597 else
598 pool_state = SPA_LOAD_NONE;
599 detector = NULL;
600 } else {
601 (void) nvlist_lookup_nvlist(nvl,
602 FM_EREPORT_DETECTOR, &detector);
603 (void) nvlist_lookup_int32(nvl,
604 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
605 }
606
607 /*
608 * We also ignore all ereports generated during an import of a pool,
609 * since the only possible fault (.pool) would result in import failure,
610 * and hence no persistent fault. Some day we may want to do something
611 * with these ereports, so we continue generating them internally.
612 */
613 if (pool_state == SPA_LOAD_IMPORT) {
614 zfs_stats.import_drops.fmds_value.ui64++;
615 fmd_hdl_debug(hdl, "ignoring '%s' during import", class);
616 return;
617 }
618
619 /*
620 * Device I/O errors are ignored during pool open.
621 */
622 if (pool_state == SPA_LOAD_OPEN &&
623 (fmd_nvl_class_match(hdl, nvl,
624 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
625 fmd_nvl_class_match(hdl, nvl,
626 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
627 fmd_nvl_class_match(hdl, nvl,
628 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
629 fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class);
630 zfs_stats.dev_drops.fmds_value.ui64++;
631 return;
632 }
633
634 /*
635 * We ignore ereports for anything except disks and files.
636 */
637 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
638 &type) == 0) {
639 if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
640 strcmp(type, VDEV_TYPE_FILE) != 0) {
641 zfs_stats.vdev_drops.fmds_value.ui64++;
642 return;
643 }
644 }
645
646 /*
647 * Determine if this ereport corresponds to an open case.
648 * Each vdev or pool can have a single case.
649 */
650 (void) nvlist_lookup_uint64(nvl,
651 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
652 if (nvlist_lookup_uint64(nvl,
653 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
654 vdev_guid = 0;
655 if (nvlist_lookup_uint64(nvl,
656 FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID, &parent_guid) != 0)
657 parent_guid = 0;
658 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
659 ena = 0;
660
661 zfs_ereport_when(hdl, nvl, &er_when);
662
663 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
664 zcp = uu_list_next(zfs_cases, zcp)) {
665 if (zcp->zc_data.zc_pool_guid == pool_guid) {
666 pool_found = B_TRUE;
667 pool_load = zcp->zc_when;
668 }
669 if (zcp->zc_data.zc_vdev_guid == vdev_guid)
670 break;
671 }
672
673 /*
674 * Avoid falsely accusing a pool of being faulty. Do so by
675 * not replaying ereports that were generated prior to the
676 * current import. If the failure that generated them was
677 * transient because the device was actually removed but we
678 * didn't receive the normal asynchronous notification, we
679 * don't want to mark it as faulted and potentially panic. If
680 * there is still a problem we'd expect not to be able to
681 * import the pool, or that new ereports will be generated
682 * once the pool is used.
683 */
684 if (pool_found && timeval_earlier(&er_when, &pool_load)) {
685 fmd_hdl_debug(hdl, "ignoring pool %llx, "
686 "ereport time %lld.%lld, pool load time = %lld.%lld",
687 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
688 pool_load.ertv_sec, pool_load.ertv_nsec);
689 zfs_stats.old_drops.fmds_value.ui64++;
690 return;
691 }
692
693 if (!pool_found) {
694 /*
695 * Haven't yet seen this pool, but same situation
696 * may apply.
697 */
698 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
699 struct load_time_arg la;
700
701 la.lt_guid = pool_guid;
702 la.lt_time = &pool_load;
703 la.lt_found = B_FALSE;
704
705 if (zhdl != NULL &&
706 zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
707 la.lt_found == B_TRUE) {
708 pool_found = B_TRUE;
709
710 if (timeval_earlier(&er_when, &pool_load)) {
711 fmd_hdl_debug(hdl, "ignoring pool %llx, "
712 "ereport time %lld.%lld, "
713 "pool load time = %lld.%lld",
714 pool_guid, er_when.ertv_sec,
715 er_when.ertv_nsec, pool_load.ertv_sec,
716 pool_load.ertv_nsec);
717 zfs_stats.old_drops.fmds_value.ui64++;
718 return;
719 }
720 }
721 }
722
723 if (zcp == NULL) {
724 fmd_case_t *cs;
725 zfs_case_data_t data = { 0 };
726
727 /*
728 * If this is one of our 'fake' resource ereports, and there is
729 * no case open, simply discard it.
730 */
731 if (isresource) {
732 zfs_stats.resource_drops.fmds_value.ui64++;
733 fmd_hdl_debug(hdl, "discarding '%s for vdev %llu",
734 class, vdev_guid);
735 return;
736 }
737
738 /*
739 * Skip tracking some ereports
740 */
741 if (strcmp(class,
742 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 ||
743 strcmp(class,
744 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0) {
745 zfs_stats.resource_drops.fmds_value.ui64++;
746 return;
747 }
748
749 /*
750 * Open a new case.
751 */
752 cs = fmd_case_open(hdl, NULL);
753
754 fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'",
755 vdev_guid, class);
756
757 /*
758 * Initialize the case buffer. To commonize code, we actually
759 * create the buffer with existing data, and then call
760 * zfs_case_unserialize() to instantiate the in-core structure.
761 */
762 fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t));
763
764 data.zc_version = CASE_DATA_VERSION_SERD;
765 data.zc_ena = ena;
766 data.zc_pool_guid = pool_guid;
767 data.zc_vdev_guid = vdev_guid;
768 data.zc_parent_guid = parent_guid;
769 data.zc_pool_state = (int)pool_state;
770
771 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
772
773 zcp = zfs_case_unserialize(hdl, cs);
774 assert(zcp != NULL);
775 if (pool_found)
776 zcp->zc_when = pool_load;
777 }
778
779 if (isresource) {
780 fmd_hdl_debug(hdl, "resource event '%s'", class);
781
782 if (fmd_nvl_class_match(hdl, nvl,
783 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
784 /*
785 * The 'resource.fs.zfs.autoreplace' event indicates
786 * that the pool was loaded with the 'autoreplace'
787 * property set. In this case, any pending device
788 * failures should be ignored, as the asynchronous
789 * autoreplace handling will take care of them.
790 */
791 fmd_case_close(hdl, zcp->zc_case);
792 } else if (fmd_nvl_class_match(hdl, nvl,
793 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
794 /*
795 * The 'resource.fs.zfs.removed' event indicates that
796 * device removal was detected, and the device was
797 * closed asynchronously. If this is the case, we
798 * assume that any recent I/O errors were due to the
799 * device removal, not any fault of the device itself.
800 * We reset the SERD engine, and cancel any pending
801 * timers.
802 */
803 if (zcp->zc_data.zc_has_remove_timer) {
804 fmd_timer_remove(hdl, zcp->zc_remove_timer);
805 zcp->zc_data.zc_has_remove_timer = 0;
806 zfs_case_serialize(zcp);
807 }
808 if (zcp->zc_data.zc_serd_io[0] != '\0')
809 fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io);
810 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
811 fmd_serd_reset(hdl,
812 zcp->zc_data.zc_serd_checksum);
813 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
814 fmd_serd_reset(hdl,
815 zcp->zc_data.zc_serd_slow_io);
816 } else if (fmd_nvl_class_match(hdl, nvl,
817 ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) {
818 uint64_t state = 0;
819
820 if (zcp != NULL &&
821 nvlist_lookup_uint64(nvl,
822 FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 &&
823 state == VDEV_STATE_HEALTHY) {
824 fmd_hdl_debug(hdl, "closing case after a "
825 "device statechange to healthy");
826 fmd_case_close(hdl, zcp->zc_case);
827 }
828 }
829 zfs_stats.resource_drops.fmds_value.ui64++;
830 return;
831 }
832
833 /*
834 * Associate the ereport with this case.
835 */
836 fmd_case_add_ereport(hdl, zcp->zc_case, ep);
837
838 /*
839 * Don't do anything else if this case is already solved.
840 */
841 if (fmd_case_solved(hdl, zcp->zc_case))
842 return;
843
844 if (vdev_guid)
845 fmd_hdl_debug(hdl, "error event '%s', vdev %llu", class,
846 vdev_guid);
847 else
848 fmd_hdl_debug(hdl, "error event '%s'", class);
849
850 /*
851 * Determine if we should solve the case and generate a fault. We solve
852 * a case if:
853 *
854 * a. A pool failed to open (ereport.fs.zfs.pool)
855 * b. A device failed to open (ereport.fs.zfs.pool) while a pool
856 * was up and running.
857 *
858 * We may see a series of ereports associated with a pool open, all
859 * chained together by the same ENA. If the pool open succeeds, then
860 * we'll see no further ereports. To detect when a pool open has
861 * succeeded, we associate a timer with the event. When it expires, we
862 * close the case.
863 */
864 if (fmd_nvl_class_match(hdl, nvl,
865 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
866 /*
867 * Pool level fault. Before solving the case, go through and
868 * close any open device cases that may be pending.
869 */
870 for (dcp = uu_list_first(zfs_cases); dcp != NULL;
871 dcp = uu_list_next(zfs_cases, dcp)) {
872 if (dcp->zc_data.zc_pool_guid ==
873 zcp->zc_data.zc_pool_guid &&
874 dcp->zc_data.zc_vdev_guid != 0)
875 fmd_case_close(hdl, dcp->zc_case);
876 }
877
878 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool");
879 } else if (fmd_nvl_class_match(hdl, nvl,
880 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
881 /*
882 * Pool level fault for reading the intent logs.
883 */
884 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay");
885 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
886 /*
887 * Device fault.
888 */
889 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device");
890 } else if (fmd_nvl_class_match(hdl, nvl,
891 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
892 fmd_nvl_class_match(hdl, nvl,
893 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
894 fmd_nvl_class_match(hdl, nvl,
895 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
896 fmd_nvl_class_match(hdl, nvl,
897 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) ||
898 fmd_nvl_class_match(hdl, nvl,
899 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
900 const char *failmode = NULL;
901 boolean_t checkremove = B_FALSE;
902 uint32_t pri = 0;
903
904 /*
905 * If this is a checksum or I/O error, then toss it into the
906 * appropriate SERD engine and check to see if it has fired.
907 * Ideally, we want to do something more sophisticated,
908 * (persistent errors for a single data block, etc). For now,
909 * a single SERD engine is sufficient.
910 */
911 if (fmd_nvl_class_match(hdl, nvl,
912 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
913 if (zcp->zc_data.zc_serd_io[0] == '\0') {
914 if (nvlist_lookup_uint64(nvl,
915 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
916 &io_n) != 0) {
917 io_n = DEFAULT_IO_N;
918 }
919 if (nvlist_lookup_uint64(nvl,
920 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
921 &io_t) != 0) {
922 io_t = DEFAULT_IO_T;
923 }
924 zfs_serd_name(zcp->zc_data.zc_serd_io,
925 pool_guid, vdev_guid, "io");
926 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
927 io_n,
928 SEC2NSEC(io_t));
929 zfs_case_serialize(zcp);
930 }
931 if (zfs_fm_serd_record(hdl, zcp->zc_data.zc_serd_io,
932 ep, zcp, "io error")) {
933 checkremove = B_TRUE;
934 }
935 } else if (fmd_nvl_class_match(hdl, nvl,
936 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY))) {
937 uint64_t slow_io_n, slow_io_t;
938
939 /*
940 * Create a slow io SERD engine when the VDEV has the
941 * 'vdev_slow_io_n' and 'vdev_slow_io_n' properties.
942 */
943 if (zcp->zc_data.zc_serd_slow_io[0] == '\0' &&
944 nvlist_lookup_uint64(nvl,
945 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N,
946 &slow_io_n) == 0 &&
947 nvlist_lookup_uint64(nvl,
948 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T,
949 &slow_io_t) == 0) {
950 zfs_serd_name(zcp->zc_data.zc_serd_slow_io,
951 pool_guid, vdev_guid, "slow_io");
952 fmd_serd_create(hdl,
953 zcp->zc_data.zc_serd_slow_io,
954 slow_io_n,
955 SEC2NSEC(slow_io_t));
956 zfs_case_serialize(zcp);
957 }
958 /* Pass event to SERD engine and see if this triggers */
959 if (zcp->zc_data.zc_serd_slow_io[0] != '\0' &&
960 zfs_fm_serd_record(hdl,
961 zcp->zc_data.zc_serd_slow_io, ep, zcp, "slow io")) {
962 zfs_case_solve(hdl, zcp,
963 "fault.fs.zfs.vdev.slow_io");
964 }
965 } else if (fmd_nvl_class_match(hdl, nvl,
966 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
967 uint64_t flags = 0;
968 int32_t flags32 = 0;
969 /*
970 * We ignore ereports for checksum errors generated by
971 * scrub/resilver I/O to avoid potentially further
972 * degrading the pool while it's being repaired.
973 *
974 * Note that FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS used to
975 * be int32. To allow newer zed to work on older
976 * kernels, if we don't find the flags, we look for
977 * the older ones too.
978 */
979 if (((nvlist_lookup_uint32(nvl,
980 FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY, &pri) == 0) &&
981 (pri == ZIO_PRIORITY_SCRUB ||
982 pri == ZIO_PRIORITY_REBUILD)) ||
983 ((nvlist_lookup_uint64(nvl,
984 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags) == 0) &&
985 (flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) ||
986 ((nvlist_lookup_int32(nvl,
987 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags32) == 0) &&
988 (flags32 & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)))) {
989 fmd_hdl_debug(hdl, "ignoring '%s' for "
990 "scrub/resilver I/O", class);
991 return;
992 }
993
994 if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
995 if (nvlist_lookup_uint64(nvl,
996 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
997 &checksum_n) != 0) {
998 checksum_n = DEFAULT_CHECKSUM_N;
999 }
1000 if (nvlist_lookup_uint64(nvl,
1001 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
1002 &checksum_t) != 0) {
1003 checksum_t = DEFAULT_CHECKSUM_T;
1004 }
1005
1006 zfs_serd_name(zcp->zc_data.zc_serd_checksum,
1007 pool_guid, vdev_guid, "checksum");
1008 fmd_serd_create(hdl,
1009 zcp->zc_data.zc_serd_checksum,
1010 checksum_n,
1011 SEC2NSEC(checksum_t));
1012 zfs_case_serialize(zcp);
1013 }
1014 if (zfs_fm_serd_record(hdl,
1015 zcp->zc_data.zc_serd_checksum, ep, zcp,
1016 "checksum")) {
1017 zfs_case_solve(hdl, zcp,
1018 "fault.fs.zfs.vdev.checksum");
1019 }
1020 } else if (fmd_nvl_class_match(hdl, nvl,
1021 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
1022 (nvlist_lookup_string(nvl,
1023 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
1024 failmode != NULL) {
1025 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
1026 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
1027 zfs_case_solve(hdl, zcp,
1028 "fault.fs.zfs.io_failure_continue");
1029 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
1030 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
1031 zfs_case_solve(hdl, zcp,
1032 "fault.fs.zfs.io_failure_wait");
1033 }
1034 } else if (fmd_nvl_class_match(hdl, nvl,
1035 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
1036 #ifndef __linux__
1037 /* This causes an unexpected fault diagnosis on linux */
1038 checkremove = B_TRUE;
1039 #endif
1040 }
1041
1042 /*
1043 * Because I/O errors may be due to device removal, we postpone
1044 * any diagnosis until we're sure that we aren't about to
1045 * receive a 'resource.fs.zfs.removed' event.
1046 */
1047 if (checkremove) {
1048 if (zcp->zc_data.zc_has_remove_timer)
1049 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1050 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
1051 zfs_remove_timeout);
1052 if (!zcp->zc_data.zc_has_remove_timer) {
1053 zcp->zc_data.zc_has_remove_timer = 1;
1054 zfs_case_serialize(zcp);
1055 }
1056 }
1057 }
1058 }
1059
1060 /*
1061 * The timeout is fired when we diagnosed an I/O error, and it was not due to
1062 * device removal (which would cause the timeout to be cancelled).
1063 */
1064 static void
zfs_fm_timeout(fmd_hdl_t * hdl,id_t id,void * data)1065 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
1066 {
1067 zfs_case_t *zcp = data;
1068
1069 if (id == zcp->zc_remove_timer)
1070 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io");
1071 }
1072
1073 /*
1074 * The specified case has been closed and any case-specific
1075 * data structures should be deallocated.
1076 */
1077 static void
zfs_fm_close(fmd_hdl_t * hdl,fmd_case_t * cs)1078 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
1079 {
1080 zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
1081
1082 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
1083 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
1084 if (zcp->zc_data.zc_serd_io[0] != '\0')
1085 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
1086 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
1087 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_slow_io);
1088 if (zcp->zc_data.zc_has_remove_timer)
1089 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1090
1091 uu_list_remove(zfs_cases, zcp);
1092 uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
1093 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1094 }
1095
1096 static const fmd_hdl_ops_t fmd_ops = {
1097 zfs_fm_recv, /* fmdo_recv */
1098 zfs_fm_timeout, /* fmdo_timeout */
1099 zfs_fm_close, /* fmdo_close */
1100 NULL, /* fmdo_stats */
1101 NULL, /* fmdo_gc */
1102 };
1103
1104 static const fmd_prop_t fmd_props[] = {
1105 { NULL, 0, NULL }
1106 };
1107
1108 static const fmd_hdl_info_t fmd_info = {
1109 "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
1110 };
1111
1112 void
_zfs_diagnosis_init(fmd_hdl_t * hdl)1113 _zfs_diagnosis_init(fmd_hdl_t *hdl)
1114 {
1115 libzfs_handle_t *zhdl;
1116
1117 if ((zhdl = libzfs_init()) == NULL)
1118 return;
1119
1120 if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
1121 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
1122 NULL, UU_LIST_POOL_DEBUG)) == NULL) {
1123 libzfs_fini(zhdl);
1124 return;
1125 }
1126
1127 if ((zfs_cases = uu_list_create(zfs_case_pool, NULL,
1128 UU_LIST_DEBUG)) == NULL) {
1129 uu_list_pool_destroy(zfs_case_pool);
1130 libzfs_fini(zhdl);
1131 return;
1132 }
1133
1134 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
1135 uu_list_destroy(zfs_cases);
1136 uu_list_pool_destroy(zfs_case_pool);
1137 libzfs_fini(zhdl);
1138 return;
1139 }
1140
1141 fmd_hdl_setspecific(hdl, zhdl);
1142
1143 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
1144 sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
1145 }
1146
1147 void
_zfs_diagnosis_fini(fmd_hdl_t * hdl)1148 _zfs_diagnosis_fini(fmd_hdl_t *hdl)
1149 {
1150 zfs_case_t *zcp;
1151 uu_list_walk_t *walk;
1152 libzfs_handle_t *zhdl;
1153
1154 /*
1155 * Remove all active cases.
1156 */
1157 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
1158 while ((zcp = uu_list_walk_next(walk)) != NULL) {
1159 fmd_hdl_debug(hdl, "removing case ena %llu",
1160 (long long unsigned)zcp->zc_data.zc_ena);
1161 uu_list_remove(zfs_cases, zcp);
1162 uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
1163 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1164 }
1165 uu_list_walk_end(walk);
1166
1167 uu_list_destroy(zfs_cases);
1168 uu_list_pool_destroy(zfs_case_pool);
1169
1170 zhdl = fmd_hdl_getspecific(hdl);
1171 libzfs_fini(zhdl);
1172 }
1173