1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2016, Intel Corporation.
26 * Copyright (c) 2023, Klara Inc.
27 */
28
29 #include <stddef.h>
30 #include <string.h>
31 #include <libuutil.h>
32 #include <libzfs.h>
33 #include <sys/types.h>
34 #include <sys/time.h>
35 #include <sys/fs/zfs.h>
36 #include <sys/fm/protocol.h>
37 #include <sys/fm/fs/zfs.h>
38 #include <sys/zio.h>
39
40 #include "zfs_agents.h"
41 #include "fmd_api.h"
42
43 /*
44 * Default values for the serd engine when processing checksum or io errors. The
45 * semantics are N <events> in T <seconds>.
46 */
47 #define DEFAULT_CHECKSUM_N 10 /* events */
48 #define DEFAULT_CHECKSUM_T 600 /* seconds */
49 #define DEFAULT_IO_N 10 /* events */
50 #define DEFAULT_IO_T 600 /* seconds */
51 #define DEFAULT_SLOW_IO_N 10 /* events */
52 #define DEFAULT_SLOW_IO_T 30 /* seconds */
53
54 #define CASE_GC_TIMEOUT_SECS 43200 /* 12 hours */
55
56 /*
57 * Our serd engines are named in the following format:
58 * 'zfs_<pool_guid>_<vdev_guid>_{checksum,io,slow_io}'
59 * This #define reserves enough space for two 64-bit hex values plus the
60 * length of the longest string.
61 */
62 #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))
63
64 /*
65 * On-disk case structure. This must maintain backwards compatibility with
66 * previous versions of the DE. By default, any members appended to the end
67 * will be filled with zeros if they don't exist in a previous version.
68 */
69 typedef struct zfs_case_data {
70 uint64_t zc_version;
71 uint64_t zc_ena;
72 uint64_t zc_pool_guid;
73 uint64_t zc_vdev_guid;
74 uint64_t zc_parent_guid;
75 int zc_pool_state;
76 char zc_serd_checksum[MAX_SERDLEN];
77 char zc_serd_io[MAX_SERDLEN];
78 char zc_serd_slow_io[MAX_SERDLEN];
79 int zc_has_remove_timer;
80 } zfs_case_data_t;
81
82 /*
83 * Time-of-day
84 */
85 typedef struct er_timeval {
86 uint64_t ertv_sec;
87 uint64_t ertv_nsec;
88 } er_timeval_t;
89
90 /*
91 * In-core case structure.
92 */
93 typedef struct zfs_case {
94 boolean_t zc_present;
95 uint32_t zc_version;
96 zfs_case_data_t zc_data;
97 fmd_case_t *zc_case;
98 uu_list_node_t zc_node;
99 id_t zc_remove_timer;
100 char *zc_fru;
101 er_timeval_t zc_when;
102 } zfs_case_t;
103
104 #define CASE_DATA "data"
105 #define CASE_FRU "fru"
106 #define CASE_DATA_VERSION_INITIAL 1
107 #define CASE_DATA_VERSION_SERD 2
108
109 typedef struct zfs_de_stats {
110 fmd_stat_t old_drops;
111 fmd_stat_t dev_drops;
112 fmd_stat_t vdev_drops;
113 fmd_stat_t import_drops;
114 fmd_stat_t resource_drops;
115 } zfs_de_stats_t;
116
117 zfs_de_stats_t zfs_stats = {
118 { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
119 { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
120 { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
121 { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
122 { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
123 };
124
125 /* wait 15 seconds after a removal */
126 static hrtime_t zfs_remove_timeout = SEC2NSEC(15);
127
128 uu_list_pool_t *zfs_case_pool;
129 uu_list_t *zfs_cases;
130
131 #define ZFS_MAKE_RSRC(type) \
132 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
133 #define ZFS_MAKE_EREPORT(type) \
134 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
135
136 static void zfs_purge_cases(fmd_hdl_t *hdl);
137
138 /*
139 * Write out the persistent representation of an active case.
140 */
141 static void
zfs_case_serialize(zfs_case_t * zcp)142 zfs_case_serialize(zfs_case_t *zcp)
143 {
144 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
145 }
146
147 /*
148 * Read back the persistent representation of an active case.
149 */
150 static zfs_case_t *
zfs_case_unserialize(fmd_hdl_t * hdl,fmd_case_t * cp)151 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
152 {
153 zfs_case_t *zcp;
154
155 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
156 zcp->zc_case = cp;
157
158 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
159 sizeof (zcp->zc_data));
160
161 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
162 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
163 return (NULL);
164 }
165
166 /*
167 * fmd_buf_read() will have already zeroed out the remainder of the
168 * buffer, so we don't have to do anything special if the version
169 * doesn't include the SERD engine name.
170 */
171
172 if (zcp->zc_data.zc_has_remove_timer)
173 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
174 NULL, zfs_remove_timeout);
175
176 uu_list_node_init(zcp, &zcp->zc_node, zfs_case_pool);
177 (void) uu_list_insert_before(zfs_cases, NULL, zcp);
178
179 fmd_case_setspecific(hdl, cp, zcp);
180
181 return (zcp);
182 }
183
184 /*
185 * Return count of other unique SERD cases under same vdev parent
186 */
187 static uint_t
zfs_other_serd_cases(fmd_hdl_t * hdl,const zfs_case_data_t * zfs_case)188 zfs_other_serd_cases(fmd_hdl_t *hdl, const zfs_case_data_t *zfs_case)
189 {
190 zfs_case_t *zcp;
191 uint_t cases = 0;
192 static hrtime_t next_check = 0;
193
194 /*
195 * Note that plumbing in some external GC would require adding locking,
196 * since most of this module code is not thread safe and assumes there
197 * is only one thread running against the module. So we perform GC here
198 * inline periodically so that future delay induced faults will be
199 * possible once the issue causing multiple vdev delays is resolved.
200 */
201 if (gethrestime_sec() > next_check) {
202 /* Periodically purge old SERD entries and stale cases */
203 fmd_serd_gc(hdl);
204 zfs_purge_cases(hdl);
205 next_check = gethrestime_sec() + CASE_GC_TIMEOUT_SECS;
206 }
207
208 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
209 zcp = uu_list_next(zfs_cases, zcp)) {
210 zfs_case_data_t *zcd = &zcp->zc_data;
211
212 /*
213 * must be same pool and parent vdev but different leaf vdev
214 */
215 if (zcd->zc_pool_guid != zfs_case->zc_pool_guid ||
216 zcd->zc_parent_guid != zfs_case->zc_parent_guid ||
217 zcd->zc_vdev_guid == zfs_case->zc_vdev_guid) {
218 continue;
219 }
220
221 /*
222 * Check if there is another active serd case besides zfs_case
223 *
224 * Only one serd engine will be assigned to the case
225 */
226 if (zcd->zc_serd_checksum[0] == zfs_case->zc_serd_checksum[0] &&
227 fmd_serd_active(hdl, zcd->zc_serd_checksum)) {
228 cases++;
229 }
230 if (zcd->zc_serd_io[0] == zfs_case->zc_serd_io[0] &&
231 fmd_serd_active(hdl, zcd->zc_serd_io)) {
232 cases++;
233 }
234 if (zcd->zc_serd_slow_io[0] == zfs_case->zc_serd_slow_io[0] &&
235 fmd_serd_active(hdl, zcd->zc_serd_slow_io)) {
236 cases++;
237 }
238 }
239 return (cases);
240 }
241
242 /*
243 * Iterate over any active cases. If any cases are associated with a pool or
244 * vdev which is no longer present on the system, close the associated case.
245 */
246 static void
zfs_mark_vdev(uint64_t pool_guid,nvlist_t * vd,er_timeval_t * loaded)247 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
248 {
249 uint64_t vdev_guid = 0;
250 uint_t c, children;
251 nvlist_t **child;
252 zfs_case_t *zcp;
253
254 (void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
255
256 /*
257 * Mark any cases associated with this (pool, vdev) pair.
258 */
259 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
260 zcp = uu_list_next(zfs_cases, zcp)) {
261 if (zcp->zc_data.zc_pool_guid == pool_guid &&
262 zcp->zc_data.zc_vdev_guid == vdev_guid) {
263 zcp->zc_present = B_TRUE;
264 zcp->zc_when = *loaded;
265 }
266 }
267
268 /*
269 * Iterate over all children.
270 */
271 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
272 &children) == 0) {
273 for (c = 0; c < children; c++)
274 zfs_mark_vdev(pool_guid, child[c], loaded);
275 }
276
277 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
278 &children) == 0) {
279 for (c = 0; c < children; c++)
280 zfs_mark_vdev(pool_guid, child[c], loaded);
281 }
282
283 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
284 &children) == 0) {
285 for (c = 0; c < children; c++)
286 zfs_mark_vdev(pool_guid, child[c], loaded);
287 }
288 }
289
290 static int
zfs_mark_pool(zpool_handle_t * zhp,void * unused)291 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
292 {
293 (void) unused;
294 zfs_case_t *zcp;
295 uint64_t pool_guid;
296 uint64_t *tod;
297 er_timeval_t loaded = { 0 };
298 nvlist_t *config, *vd;
299 uint_t nelem = 0;
300 int ret;
301
302 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
303 /*
304 * Mark any cases associated with just this pool.
305 */
306 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
307 zcp = uu_list_next(zfs_cases, zcp)) {
308 if (zcp->zc_data.zc_pool_guid == pool_guid &&
309 zcp->zc_data.zc_vdev_guid == 0)
310 zcp->zc_present = B_TRUE;
311 }
312
313 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
314 zpool_close(zhp);
315 return (-1);
316 }
317
318 (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
319 &tod, &nelem);
320 if (nelem == 2) {
321 loaded.ertv_sec = tod[0];
322 loaded.ertv_nsec = tod[1];
323 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
324 zcp = uu_list_next(zfs_cases, zcp)) {
325 if (zcp->zc_data.zc_pool_guid == pool_guid &&
326 zcp->zc_data.zc_vdev_guid == 0) {
327 zcp->zc_when = loaded;
328 }
329 }
330 }
331
332 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
333 if (ret) {
334 zpool_close(zhp);
335 return (-1);
336 }
337
338 zfs_mark_vdev(pool_guid, vd, &loaded);
339
340 zpool_close(zhp);
341
342 return (0);
343 }
344
345 struct load_time_arg {
346 uint64_t lt_guid;
347 er_timeval_t *lt_time;
348 boolean_t lt_found;
349 };
350
351 static int
zpool_find_load_time(zpool_handle_t * zhp,void * arg)352 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
353 {
354 struct load_time_arg *lta = arg;
355 uint64_t pool_guid;
356 uint64_t *tod;
357 nvlist_t *config;
358 uint_t nelem;
359
360 if (lta->lt_found) {
361 zpool_close(zhp);
362 return (0);
363 }
364
365 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
366 if (pool_guid != lta->lt_guid) {
367 zpool_close(zhp);
368 return (0);
369 }
370
371 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
372 zpool_close(zhp);
373 return (-1);
374 }
375
376 if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
377 &tod, &nelem) == 0 && nelem == 2) {
378 lta->lt_found = B_TRUE;
379 lta->lt_time->ertv_sec = tod[0];
380 lta->lt_time->ertv_nsec = tod[1];
381 }
382
383 zpool_close(zhp);
384
385 return (0);
386 }
387
388 static void
zfs_purge_cases(fmd_hdl_t * hdl)389 zfs_purge_cases(fmd_hdl_t *hdl)
390 {
391 zfs_case_t *zcp;
392 uu_list_walk_t *walk;
393 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
394
395 /*
396 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No
397 * matter what we do, we're going to have to stomach an O(vdevs * cases)
398 * algorithm. In reality, both quantities are likely so small that
399 * neither will matter. Given that iterating over pools is more
400 * expensive than iterating over the in-memory case list, we opt for a
401 * 'present' flag in each case that starts off cleared. We then iterate
402 * over all pools, marking those that are still present, and removing
403 * those that aren't found.
404 *
405 * Note that we could also construct an FMRI and rely on
406 * fmd_nvl_fmri_present(), but this would end up doing the same search.
407 */
408
409 /*
410 * Mark the cases as not present.
411 */
412 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
413 zcp = uu_list_next(zfs_cases, zcp))
414 zcp->zc_present = B_FALSE;
415
416 /*
417 * Iterate over all pools and mark the pools and vdevs found. If this
418 * fails (most probably because we're out of memory), then don't close
419 * any of the cases and we cannot be sure they are accurate.
420 */
421 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
422 return;
423
424 /*
425 * Remove those cases which were not found.
426 */
427 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
428 while ((zcp = uu_list_walk_next(walk)) != NULL) {
429 if (!zcp->zc_present)
430 fmd_case_close(hdl, zcp->zc_case);
431 }
432 uu_list_walk_end(walk);
433 }
434
435 /*
436 * Construct the name of a serd engine given the pool/vdev GUID and type (io or
437 * checksum).
438 */
439 static void
zfs_serd_name(char * buf,uint64_t pool_guid,uint64_t vdev_guid,const char * type)440 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
441 const char *type)
442 {
443 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s",
444 (long long unsigned int)pool_guid,
445 (long long unsigned int)vdev_guid, type);
446 }
447
448 static void
zfs_case_retire(fmd_hdl_t * hdl,zfs_case_t * zcp)449 zfs_case_retire(fmd_hdl_t *hdl, zfs_case_t *zcp)
450 {
451 fmd_hdl_debug(hdl, "retiring case");
452
453 fmd_case_close(hdl, zcp->zc_case);
454 }
455
456 /*
457 * Solve a given ZFS case. This first checks to make sure the diagnosis is
458 * still valid, as well as cleaning up any pending timer associated with the
459 * case.
460 */
461 static void
zfs_case_solve(fmd_hdl_t * hdl,zfs_case_t * zcp,const char * faultname)462 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname)
463 {
464 nvlist_t *detector, *fault;
465 boolean_t serialize;
466 nvlist_t *fru = NULL;
467 fmd_hdl_debug(hdl, "solving fault '%s'", faultname);
468
469 /*
470 * Construct the detector from the case data. The detector is in the
471 * ZFS scheme, and is either the pool or the vdev, depending on whether
472 * this is a vdev or pool fault.
473 */
474 detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
475
476 (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
477 (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
478 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
479 zcp->zc_data.zc_pool_guid);
480 if (zcp->zc_data.zc_vdev_guid != 0) {
481 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
482 zcp->zc_data.zc_vdev_guid);
483 }
484
485 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
486 fru, detector);
487 fmd_case_add_suspect(hdl, zcp->zc_case, fault);
488
489 nvlist_free(fru);
490
491 fmd_case_solve(hdl, zcp->zc_case);
492
493 serialize = B_FALSE;
494 if (zcp->zc_data.zc_has_remove_timer) {
495 fmd_timer_remove(hdl, zcp->zc_remove_timer);
496 zcp->zc_data.zc_has_remove_timer = 0;
497 serialize = B_TRUE;
498 }
499 if (serialize)
500 zfs_case_serialize(zcp);
501
502 nvlist_free(detector);
503 }
504
505 static boolean_t
timeval_earlier(er_timeval_t * a,er_timeval_t * b)506 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
507 {
508 return (a->ertv_sec < b->ertv_sec ||
509 (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
510 }
511
512 static void
zfs_ereport_when(fmd_hdl_t * hdl,nvlist_t * nvl,er_timeval_t * when)513 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
514 {
515 (void) hdl;
516 int64_t *tod;
517 uint_t nelem;
518
519 if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod,
520 &nelem) == 0 && nelem == 2) {
521 when->ertv_sec = tod[0];
522 when->ertv_nsec = tod[1];
523 } else {
524 when->ertv_sec = when->ertv_nsec = UINT64_MAX;
525 }
526 }
527
528 /*
529 * Record the specified event in the SERD engine and return a
530 * boolean value indicating whether or not the engine fired as
531 * the result of inserting this event.
532 *
533 * When the pool has similar active cases on other vdevs, then
534 * the fired state is disregarded and the case is retired.
535 */
536 static int
zfs_fm_serd_record(fmd_hdl_t * hdl,const char * name,fmd_event_t * ep,zfs_case_t * zcp,const char * err_type)537 zfs_fm_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep,
538 zfs_case_t *zcp, const char *err_type)
539 {
540 int fired = fmd_serd_record(hdl, name, ep);
541 int peers = 0;
542
543 if (fired && (peers = zfs_other_serd_cases(hdl, &zcp->zc_data)) > 0) {
544 fmd_hdl_debug(hdl, "pool %llu is tracking %d other %s cases "
545 "-- skip faulting the vdev %llu",
546 (u_longlong_t)zcp->zc_data.zc_pool_guid,
547 peers, err_type,
548 (u_longlong_t)zcp->zc_data.zc_vdev_guid);
549 zfs_case_retire(hdl, zcp);
550 fired = 0;
551 }
552
553 return (fired);
554 }
555
556 /*
557 * Main fmd entry point.
558 */
559 static void
zfs_fm_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)560 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
561 {
562 zfs_case_t *zcp, *dcp;
563 int32_t pool_state;
564 uint64_t ena, pool_guid, vdev_guid, parent_guid;
565 uint64_t checksum_n, checksum_t;
566 uint64_t io_n, io_t;
567 er_timeval_t pool_load;
568 er_timeval_t er_when;
569 nvlist_t *detector;
570 boolean_t pool_found = B_FALSE;
571 boolean_t isresource;
572 const char *type;
573
574 /*
575 * We subscribe to notifications for vdev or pool removal. In these
576 * cases, there may be cases that no longer apply. Purge any cases
577 * that no longer apply.
578 */
579 if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) {
580 fmd_hdl_debug(hdl, "purging orphaned cases from %s",
581 strrchr(class, '.') + 1);
582 zfs_purge_cases(hdl);
583 zfs_stats.resource_drops.fmds_value.ui64++;
584 return;
585 }
586
587 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
588
589 if (isresource) {
590 /*
591 * For resources, we don't have a normal payload.
592 */
593 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
594 &vdev_guid) != 0)
595 pool_state = SPA_LOAD_OPEN;
596 else
597 pool_state = SPA_LOAD_NONE;
598 detector = NULL;
599 } else {
600 (void) nvlist_lookup_nvlist(nvl,
601 FM_EREPORT_DETECTOR, &detector);
602 (void) nvlist_lookup_int32(nvl,
603 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
604 }
605
606 /*
607 * We also ignore all ereports generated during an import of a pool,
608 * since the only possible fault (.pool) would result in import failure,
609 * and hence no persistent fault. Some day we may want to do something
610 * with these ereports, so we continue generating them internally.
611 */
612 if (pool_state == SPA_LOAD_IMPORT) {
613 zfs_stats.import_drops.fmds_value.ui64++;
614 fmd_hdl_debug(hdl, "ignoring '%s' during import", class);
615 return;
616 }
617
618 /*
619 * Device I/O errors are ignored during pool open.
620 */
621 if (pool_state == SPA_LOAD_OPEN &&
622 (fmd_nvl_class_match(hdl, nvl,
623 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
624 fmd_nvl_class_match(hdl, nvl,
625 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
626 fmd_nvl_class_match(hdl, nvl,
627 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
628 fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class);
629 zfs_stats.dev_drops.fmds_value.ui64++;
630 return;
631 }
632
633 /*
634 * We ignore ereports for anything except disks and files.
635 */
636 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
637 &type) == 0) {
638 if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
639 strcmp(type, VDEV_TYPE_FILE) != 0) {
640 zfs_stats.vdev_drops.fmds_value.ui64++;
641 return;
642 }
643 }
644
645 /*
646 * Determine if this ereport corresponds to an open case.
647 * Each vdev or pool can have a single case.
648 */
649 (void) nvlist_lookup_uint64(nvl,
650 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
651 if (nvlist_lookup_uint64(nvl,
652 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
653 vdev_guid = 0;
654 if (nvlist_lookup_uint64(nvl,
655 FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID, &parent_guid) != 0)
656 parent_guid = 0;
657 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
658 ena = 0;
659
660 zfs_ereport_when(hdl, nvl, &er_when);
661
662 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
663 zcp = uu_list_next(zfs_cases, zcp)) {
664 if (zcp->zc_data.zc_pool_guid == pool_guid) {
665 pool_found = B_TRUE;
666 pool_load = zcp->zc_when;
667 }
668 if (zcp->zc_data.zc_vdev_guid == vdev_guid)
669 break;
670 }
671
672 /*
673 * Avoid falsely accusing a pool of being faulty. Do so by
674 * not replaying ereports that were generated prior to the
675 * current import. If the failure that generated them was
676 * transient because the device was actually removed but we
677 * didn't receive the normal asynchronous notification, we
678 * don't want to mark it as faulted and potentially panic. If
679 * there is still a problem we'd expect not to be able to
680 * import the pool, or that new ereports will be generated
681 * once the pool is used.
682 */
683 if (pool_found && timeval_earlier(&er_when, &pool_load)) {
684 fmd_hdl_debug(hdl, "ignoring pool %llx, "
685 "ereport time %lld.%lld, pool load time = %lld.%lld",
686 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
687 pool_load.ertv_sec, pool_load.ertv_nsec);
688 zfs_stats.old_drops.fmds_value.ui64++;
689 return;
690 }
691
692 if (!pool_found) {
693 /*
694 * Haven't yet seen this pool, but same situation
695 * may apply.
696 */
697 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
698 struct load_time_arg la;
699
700 la.lt_guid = pool_guid;
701 la.lt_time = &pool_load;
702 la.lt_found = B_FALSE;
703
704 if (zhdl != NULL &&
705 zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
706 la.lt_found == B_TRUE) {
707 pool_found = B_TRUE;
708
709 if (timeval_earlier(&er_when, &pool_load)) {
710 fmd_hdl_debug(hdl, "ignoring pool %llx, "
711 "ereport time %lld.%lld, "
712 "pool load time = %lld.%lld",
713 pool_guid, er_when.ertv_sec,
714 er_when.ertv_nsec, pool_load.ertv_sec,
715 pool_load.ertv_nsec);
716 zfs_stats.old_drops.fmds_value.ui64++;
717 return;
718 }
719 }
720 }
721
722 if (zcp == NULL) {
723 fmd_case_t *cs;
724 zfs_case_data_t data = { 0 };
725
726 /*
727 * If this is one of our 'fake' resource ereports, and there is
728 * no case open, simply discard it.
729 */
730 if (isresource) {
731 zfs_stats.resource_drops.fmds_value.ui64++;
732 fmd_hdl_debug(hdl, "discarding '%s for vdev %llu",
733 class, vdev_guid);
734 return;
735 }
736
737 /*
738 * Skip tracking some ereports
739 */
740 if (strcmp(class,
741 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 ||
742 strcmp(class,
743 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0) {
744 zfs_stats.resource_drops.fmds_value.ui64++;
745 return;
746 }
747
748 /*
749 * Open a new case.
750 */
751 cs = fmd_case_open(hdl, NULL);
752
753 fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'",
754 vdev_guid, class);
755
756 /*
757 * Initialize the case buffer. To commonize code, we actually
758 * create the buffer with existing data, and then call
759 * zfs_case_unserialize() to instantiate the in-core structure.
760 */
761 fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t));
762
763 data.zc_version = CASE_DATA_VERSION_SERD;
764 data.zc_ena = ena;
765 data.zc_pool_guid = pool_guid;
766 data.zc_vdev_guid = vdev_guid;
767 data.zc_parent_guid = parent_guid;
768 data.zc_pool_state = (int)pool_state;
769
770 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
771
772 zcp = zfs_case_unserialize(hdl, cs);
773 assert(zcp != NULL);
774 if (pool_found)
775 zcp->zc_when = pool_load;
776 }
777
778 if (isresource) {
779 fmd_hdl_debug(hdl, "resource event '%s'", class);
780
781 if (fmd_nvl_class_match(hdl, nvl,
782 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
783 /*
784 * The 'resource.fs.zfs.autoreplace' event indicates
785 * that the pool was loaded with the 'autoreplace'
786 * property set. In this case, any pending device
787 * failures should be ignored, as the asynchronous
788 * autoreplace handling will take care of them.
789 */
790 fmd_case_close(hdl, zcp->zc_case);
791 } else if (fmd_nvl_class_match(hdl, nvl,
792 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
793 /*
794 * The 'resource.fs.zfs.removed' event indicates that
795 * device removal was detected, and the device was
796 * closed asynchronously. If this is the case, we
797 * assume that any recent I/O errors were due to the
798 * device removal, not any fault of the device itself.
799 * We reset the SERD engine, and cancel any pending
800 * timers.
801 */
802 if (zcp->zc_data.zc_has_remove_timer) {
803 fmd_timer_remove(hdl, zcp->zc_remove_timer);
804 zcp->zc_data.zc_has_remove_timer = 0;
805 zfs_case_serialize(zcp);
806 }
807 if (zcp->zc_data.zc_serd_io[0] != '\0')
808 fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io);
809 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
810 fmd_serd_reset(hdl,
811 zcp->zc_data.zc_serd_checksum);
812 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
813 fmd_serd_reset(hdl,
814 zcp->zc_data.zc_serd_slow_io);
815 } else if (fmd_nvl_class_match(hdl, nvl,
816 ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) {
817 uint64_t state = 0;
818
819 if (zcp != NULL &&
820 nvlist_lookup_uint64(nvl,
821 FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 &&
822 state == VDEV_STATE_HEALTHY) {
823 fmd_hdl_debug(hdl, "closing case after a "
824 "device statechange to healthy");
825 fmd_case_close(hdl, zcp->zc_case);
826 }
827 }
828 zfs_stats.resource_drops.fmds_value.ui64++;
829 return;
830 }
831
832 /*
833 * Associate the ereport with this case.
834 */
835 fmd_case_add_ereport(hdl, zcp->zc_case, ep);
836
837 /*
838 * Don't do anything else if this case is already solved.
839 */
840 if (fmd_case_solved(hdl, zcp->zc_case))
841 return;
842
843 if (vdev_guid)
844 fmd_hdl_debug(hdl, "error event '%s', vdev %llu", class,
845 vdev_guid);
846 else
847 fmd_hdl_debug(hdl, "error event '%s'", class);
848
849 /*
850 * Determine if we should solve the case and generate a fault. We solve
851 * a case if:
852 *
853 * a. A pool failed to open (ereport.fs.zfs.pool)
854 * b. A device failed to open (ereport.fs.zfs.pool) while a pool
855 * was up and running.
856 *
857 * We may see a series of ereports associated with a pool open, all
858 * chained together by the same ENA. If the pool open succeeds, then
859 * we'll see no further ereports. To detect when a pool open has
860 * succeeded, we associate a timer with the event. When it expires, we
861 * close the case.
862 */
863 if (fmd_nvl_class_match(hdl, nvl,
864 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
865 /*
866 * Pool level fault. Before solving the case, go through and
867 * close any open device cases that may be pending.
868 */
869 for (dcp = uu_list_first(zfs_cases); dcp != NULL;
870 dcp = uu_list_next(zfs_cases, dcp)) {
871 if (dcp->zc_data.zc_pool_guid ==
872 zcp->zc_data.zc_pool_guid &&
873 dcp->zc_data.zc_vdev_guid != 0)
874 fmd_case_close(hdl, dcp->zc_case);
875 }
876
877 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool");
878 } else if (fmd_nvl_class_match(hdl, nvl,
879 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
880 /*
881 * Pool level fault for reading the intent logs.
882 */
883 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay");
884 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
885 /*
886 * Device fault.
887 */
888 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device");
889 } else if (fmd_nvl_class_match(hdl, nvl,
890 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
891 fmd_nvl_class_match(hdl, nvl,
892 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
893 fmd_nvl_class_match(hdl, nvl,
894 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
895 fmd_nvl_class_match(hdl, nvl,
896 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) ||
897 fmd_nvl_class_match(hdl, nvl,
898 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
899 const char *failmode = NULL;
900 boolean_t checkremove = B_FALSE;
901 uint32_t pri = 0;
902
903 /*
904 * If this is a checksum or I/O error, then toss it into the
905 * appropriate SERD engine and check to see if it has fired.
906 * Ideally, we want to do something more sophisticated,
907 * (persistent errors for a single data block, etc). For now,
908 * a single SERD engine is sufficient.
909 */
910 if (fmd_nvl_class_match(hdl, nvl,
911 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
912 if (zcp->zc_data.zc_serd_io[0] == '\0') {
913 if (nvlist_lookup_uint64(nvl,
914 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
915 &io_n) != 0) {
916 io_n = DEFAULT_IO_N;
917 }
918 if (nvlist_lookup_uint64(nvl,
919 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
920 &io_t) != 0) {
921 io_t = DEFAULT_IO_T;
922 }
923 zfs_serd_name(zcp->zc_data.zc_serd_io,
924 pool_guid, vdev_guid, "io");
925 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
926 io_n,
927 SEC2NSEC(io_t));
928 zfs_case_serialize(zcp);
929 }
930 if (zfs_fm_serd_record(hdl, zcp->zc_data.zc_serd_io,
931 ep, zcp, "io error")) {
932 checkremove = B_TRUE;
933 }
934 } else if (fmd_nvl_class_match(hdl, nvl,
935 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY))) {
936 uint64_t slow_io_n, slow_io_t;
937
938 /*
939 * Create a slow io SERD engine when the VDEV has the
940 * 'vdev_slow_io_n' and 'vdev_slow_io_n' properties.
941 */
942 if (zcp->zc_data.zc_serd_slow_io[0] == '\0' &&
943 nvlist_lookup_uint64(nvl,
944 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N,
945 &slow_io_n) == 0 &&
946 nvlist_lookup_uint64(nvl,
947 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T,
948 &slow_io_t) == 0) {
949 zfs_serd_name(zcp->zc_data.zc_serd_slow_io,
950 pool_guid, vdev_guid, "slow_io");
951 fmd_serd_create(hdl,
952 zcp->zc_data.zc_serd_slow_io,
953 slow_io_n,
954 SEC2NSEC(slow_io_t));
955 zfs_case_serialize(zcp);
956 }
957 /* Pass event to SERD engine and see if this triggers */
958 if (zcp->zc_data.zc_serd_slow_io[0] != '\0' &&
959 zfs_fm_serd_record(hdl,
960 zcp->zc_data.zc_serd_slow_io, ep, zcp, "slow io")) {
961 zfs_case_solve(hdl, zcp,
962 "fault.fs.zfs.vdev.slow_io");
963 }
964 } else if (fmd_nvl_class_match(hdl, nvl,
965 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
966 uint64_t flags = 0;
967 int32_t flags32 = 0;
968 /*
969 * We ignore ereports for checksum errors generated by
970 * scrub/resilver I/O to avoid potentially further
971 * degrading the pool while it's being repaired.
972 *
973 * Note that FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS used to
974 * be int32. To allow newer zed to work on older
975 * kernels, if we don't find the flags, we look for
976 * the older ones too.
977 */
978 if (((nvlist_lookup_uint32(nvl,
979 FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY, &pri) == 0) &&
980 (pri == ZIO_PRIORITY_SCRUB ||
981 pri == ZIO_PRIORITY_REBUILD)) ||
982 ((nvlist_lookup_uint64(nvl,
983 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags) == 0) &&
984 (flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) ||
985 ((nvlist_lookup_int32(nvl,
986 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags32) == 0) &&
987 (flags32 & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)))) {
988 fmd_hdl_debug(hdl, "ignoring '%s' for "
989 "scrub/resilver I/O", class);
990 return;
991 }
992
993 if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
994 if (nvlist_lookup_uint64(nvl,
995 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
996 &checksum_n) != 0) {
997 checksum_n = DEFAULT_CHECKSUM_N;
998 }
999 if (nvlist_lookup_uint64(nvl,
1000 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
1001 &checksum_t) != 0) {
1002 checksum_t = DEFAULT_CHECKSUM_T;
1003 }
1004
1005 zfs_serd_name(zcp->zc_data.zc_serd_checksum,
1006 pool_guid, vdev_guid, "checksum");
1007 fmd_serd_create(hdl,
1008 zcp->zc_data.zc_serd_checksum,
1009 checksum_n,
1010 SEC2NSEC(checksum_t));
1011 zfs_case_serialize(zcp);
1012 }
1013 if (zfs_fm_serd_record(hdl,
1014 zcp->zc_data.zc_serd_checksum, ep, zcp,
1015 "checksum")) {
1016 zfs_case_solve(hdl, zcp,
1017 "fault.fs.zfs.vdev.checksum");
1018 }
1019 } else if (fmd_nvl_class_match(hdl, nvl,
1020 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
1021 (nvlist_lookup_string(nvl,
1022 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
1023 failmode != NULL) {
1024 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
1025 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
1026 zfs_case_solve(hdl, zcp,
1027 "fault.fs.zfs.io_failure_continue");
1028 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
1029 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
1030 zfs_case_solve(hdl, zcp,
1031 "fault.fs.zfs.io_failure_wait");
1032 }
1033 } else if (fmd_nvl_class_match(hdl, nvl,
1034 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
1035 #ifndef __linux__
1036 /* This causes an unexpected fault diagnosis on linux */
1037 checkremove = B_TRUE;
1038 #endif
1039 }
1040
1041 /*
1042 * Because I/O errors may be due to device removal, we postpone
1043 * any diagnosis until we're sure that we aren't about to
1044 * receive a 'resource.fs.zfs.removed' event.
1045 */
1046 if (checkremove) {
1047 if (zcp->zc_data.zc_has_remove_timer)
1048 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1049 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
1050 zfs_remove_timeout);
1051 if (!zcp->zc_data.zc_has_remove_timer) {
1052 zcp->zc_data.zc_has_remove_timer = 1;
1053 zfs_case_serialize(zcp);
1054 }
1055 }
1056 }
1057 }
1058
1059 /*
1060 * The timeout is fired when we diagnosed an I/O error, and it was not due to
1061 * device removal (which would cause the timeout to be cancelled).
1062 */
1063 static void
zfs_fm_timeout(fmd_hdl_t * hdl,id_t id,void * data)1064 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
1065 {
1066 zfs_case_t *zcp = data;
1067
1068 if (id == zcp->zc_remove_timer)
1069 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io");
1070 }
1071
1072 /*
1073 * The specified case has been closed and any case-specific
1074 * data structures should be deallocated.
1075 */
1076 static void
zfs_fm_close(fmd_hdl_t * hdl,fmd_case_t * cs)1077 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
1078 {
1079 zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
1080
1081 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
1082 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
1083 if (zcp->zc_data.zc_serd_io[0] != '\0')
1084 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
1085 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
1086 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_slow_io);
1087 if (zcp->zc_data.zc_has_remove_timer)
1088 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1089
1090 uu_list_remove(zfs_cases, zcp);
1091 uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
1092 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1093 }
1094
1095 static const fmd_hdl_ops_t fmd_ops = {
1096 zfs_fm_recv, /* fmdo_recv */
1097 zfs_fm_timeout, /* fmdo_timeout */
1098 zfs_fm_close, /* fmdo_close */
1099 NULL, /* fmdo_stats */
1100 NULL, /* fmdo_gc */
1101 };
1102
1103 static const fmd_prop_t fmd_props[] = {
1104 { NULL, 0, NULL }
1105 };
1106
1107 static const fmd_hdl_info_t fmd_info = {
1108 "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
1109 };
1110
1111 void
_zfs_diagnosis_init(fmd_hdl_t * hdl)1112 _zfs_diagnosis_init(fmd_hdl_t *hdl)
1113 {
1114 libzfs_handle_t *zhdl;
1115
1116 if ((zhdl = libzfs_init()) == NULL)
1117 return;
1118
1119 if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
1120 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
1121 NULL, UU_LIST_POOL_DEBUG)) == NULL) {
1122 libzfs_fini(zhdl);
1123 return;
1124 }
1125
1126 if ((zfs_cases = uu_list_create(zfs_case_pool, NULL,
1127 UU_LIST_DEBUG)) == NULL) {
1128 uu_list_pool_destroy(zfs_case_pool);
1129 libzfs_fini(zhdl);
1130 return;
1131 }
1132
1133 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
1134 uu_list_destroy(zfs_cases);
1135 uu_list_pool_destroy(zfs_case_pool);
1136 libzfs_fini(zhdl);
1137 return;
1138 }
1139
1140 fmd_hdl_setspecific(hdl, zhdl);
1141
1142 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
1143 sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
1144 }
1145
1146 void
_zfs_diagnosis_fini(fmd_hdl_t * hdl)1147 _zfs_diagnosis_fini(fmd_hdl_t *hdl)
1148 {
1149 zfs_case_t *zcp;
1150 uu_list_walk_t *walk;
1151 libzfs_handle_t *zhdl;
1152
1153 /*
1154 * Remove all active cases.
1155 */
1156 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
1157 while ((zcp = uu_list_walk_next(walk)) != NULL) {
1158 fmd_hdl_debug(hdl, "removing case ena %llu",
1159 (long long unsigned)zcp->zc_data.zc_ena);
1160 uu_list_remove(zfs_cases, zcp);
1161 uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
1162 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1163 }
1164 uu_list_walk_end(walk);
1165
1166 uu_list_destroy(zfs_cases);
1167 uu_list_pool_destroy(zfs_case_pool);
1168
1169 zhdl = fmd_hdl_getspecific(hdl);
1170 libzfs_fini(zhdl);
1171 }
1172