1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
26 * Copyright (c) 2016, Intel Corporation.
27 * Copyright (c) 2023, Klara Inc.
28 */
29
30 #include <stddef.h>
31 #include <string.h>
32 #include <libzfs.h>
33 #include <sys/types.h>
34 #include <sys/time.h>
35 #include <sys/fs/zfs.h>
36 #include <sys/fm/protocol.h>
37 #include <sys/fm/fs/zfs.h>
38 #include <sys/zio.h>
39
40 #include "zfs_agents.h"
41 #include "fmd_api.h"
42
43 /*
44 * Default values for the serd engine when processing checksum or io errors. The
45 * semantics are N <events> in T <seconds>.
46 */
47 #define DEFAULT_CHECKSUM_N 10 /* events */
48 #define DEFAULT_CHECKSUM_T 600 /* seconds */
49 #define DEFAULT_IO_N 10 /* events */
50 #define DEFAULT_IO_T 600 /* seconds */
51 #define DEFAULT_SLOW_IO_N 10 /* events */
52 #define DEFAULT_SLOW_IO_T 30 /* seconds */
53
54 #define CASE_GC_TIMEOUT_SECS 43200 /* 12 hours */
55
56 /*
57 * Our serd engines are named in the following format:
58 * 'zfs_<pool_guid>_<vdev_guid>_{checksum,io,slow_io}'
59 * This #define reserves enough space for two 64-bit hex values plus the
60 * length of the longest string.
61 */
62 #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))
63
64 /*
65 * On-disk case structure. This must maintain backwards compatibility with
66 * previous versions of the DE. By default, any members appended to the end
67 * will be filled with zeros if they don't exist in a previous version.
68 */
69 typedef struct zfs_case_data {
70 uint64_t zc_version;
71 uint64_t zc_ena;
72 uint64_t zc_pool_guid;
73 uint64_t zc_vdev_guid;
74 uint64_t zc_parent_guid;
75 int zc_pool_state;
76 char zc_serd_checksum[MAX_SERDLEN];
77 char zc_serd_io[MAX_SERDLEN];
78 char zc_serd_slow_io[MAX_SERDLEN];
79 int zc_has_remove_timer;
80 } zfs_case_data_t;
81
82 /*
83 * Time-of-day
84 */
85 typedef struct er_timeval {
86 uint64_t ertv_sec;
87 uint64_t ertv_nsec;
88 } er_timeval_t;
89
90 /*
91 * In-core case structure.
92 */
93 typedef struct zfs_case {
94 boolean_t zc_present;
95 uint32_t zc_version;
96 zfs_case_data_t zc_data;
97 fmd_case_t *zc_case;
98 list_node_t zc_node;
99 id_t zc_remove_timer;
100 char *zc_fru;
101 er_timeval_t zc_when;
102 } zfs_case_t;
103
104 #define CASE_DATA "data"
105 #define CASE_FRU "fru"
106 #define CASE_DATA_VERSION_INITIAL 1
107 #define CASE_DATA_VERSION_SERD 2
108
109 typedef struct zfs_de_stats {
110 fmd_stat_t old_drops;
111 fmd_stat_t dev_drops;
112 fmd_stat_t vdev_drops;
113 fmd_stat_t import_drops;
114 fmd_stat_t resource_drops;
115 } zfs_de_stats_t;
116
117 zfs_de_stats_t zfs_stats = {
118 { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
119 { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
120 { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
121 { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
122 { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
123 };
124
125 /* wait 15 seconds after a removal */
126 static hrtime_t zfs_remove_timeout = SEC2NSEC(15);
127
128 static list_t zfs_cases;
129
130 #define ZFS_MAKE_RSRC(type) \
131 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
132 #define ZFS_MAKE_EREPORT(type) \
133 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
134
135 static void zfs_purge_cases(fmd_hdl_t *hdl);
136
137 /*
138 * Write out the persistent representation of an active case.
139 */
140 static void
zfs_case_serialize(zfs_case_t * zcp)141 zfs_case_serialize(zfs_case_t *zcp)
142 {
143 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
144 }
145
146 /*
147 * Read back the persistent representation of an active case.
148 */
149 static zfs_case_t *
zfs_case_unserialize(fmd_hdl_t * hdl,fmd_case_t * cp)150 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
151 {
152 zfs_case_t *zcp;
153
154 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
155 zcp->zc_case = cp;
156
157 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
158 sizeof (zcp->zc_data));
159
160 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
161 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
162 return (NULL);
163 }
164
165 /*
166 * fmd_buf_read() will have already zeroed out the remainder of the
167 * buffer, so we don't have to do anything special if the version
168 * doesn't include the SERD engine name.
169 */
170
171 if (zcp->zc_data.zc_has_remove_timer)
172 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
173 NULL, zfs_remove_timeout);
174
175 list_link_init(&zcp->zc_node);
176 list_insert_head(&zfs_cases, zcp);
177
178 fmd_case_setspecific(hdl, cp, zcp);
179
180 return (zcp);
181 }
182
183 /*
184 * Return count of other unique SERD cases under same vdev parent
185 */
186 static uint_t
zfs_other_serd_cases(fmd_hdl_t * hdl,const zfs_case_data_t * zfs_case)187 zfs_other_serd_cases(fmd_hdl_t *hdl, const zfs_case_data_t *zfs_case)
188 {
189 zfs_case_t *zcp;
190 uint_t cases = 0;
191 static hrtime_t next_check = 0;
192
193 /*
194 * Note that plumbing in some external GC would require adding locking,
195 * since most of this module code is not thread safe and assumes there
196 * is only one thread running against the module. So we perform GC here
197 * inline periodically so that future delay induced faults will be
198 * possible once the issue causing multiple vdev delays is resolved.
199 */
200 if (gethrestime_sec() > next_check) {
201 /* Periodically purge old SERD entries and stale cases */
202 fmd_serd_gc(hdl);
203 zfs_purge_cases(hdl);
204 next_check = gethrestime_sec() + CASE_GC_TIMEOUT_SECS;
205 }
206
207 for (zcp = list_head(&zfs_cases); zcp != NULL;
208 zcp = list_next(&zfs_cases, zcp)) {
209 zfs_case_data_t *zcd = &zcp->zc_data;
210
211 /*
212 * must be same pool and parent vdev but different leaf vdev
213 */
214 if (zcd->zc_pool_guid != zfs_case->zc_pool_guid ||
215 zcd->zc_parent_guid != zfs_case->zc_parent_guid ||
216 zcd->zc_vdev_guid == zfs_case->zc_vdev_guid) {
217 continue;
218 }
219
220 /*
221 * Check if there is another active serd case besides zfs_case
222 *
223 * Only one serd engine will be assigned to the case
224 */
225 if (zcd->zc_serd_checksum[0] == zfs_case->zc_serd_checksum[0] &&
226 fmd_serd_active(hdl, zcd->zc_serd_checksum)) {
227 cases++;
228 }
229 if (zcd->zc_serd_io[0] == zfs_case->zc_serd_io[0] &&
230 fmd_serd_active(hdl, zcd->zc_serd_io)) {
231 cases++;
232 }
233 if (zcd->zc_serd_slow_io[0] == zfs_case->zc_serd_slow_io[0] &&
234 fmd_serd_active(hdl, zcd->zc_serd_slow_io)) {
235 cases++;
236 }
237 }
238 return (cases);
239 }
240
241 /*
242 * Iterate over any active cases. If any cases are associated with a pool or
243 * vdev which is no longer present on the system, close the associated case.
244 */
245 static void
zfs_mark_vdev(uint64_t pool_guid,nvlist_t * vd,er_timeval_t * loaded)246 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
247 {
248 uint64_t vdev_guid = 0;
249 uint_t c, children;
250 nvlist_t **child;
251 zfs_case_t *zcp;
252
253 (void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
254
255 /*
256 * Mark any cases associated with this (pool, vdev) pair.
257 */
258 for (zcp = list_head(&zfs_cases); zcp != NULL;
259 zcp = list_next(&zfs_cases, zcp)) {
260 if (zcp->zc_data.zc_pool_guid == pool_guid &&
261 zcp->zc_data.zc_vdev_guid == vdev_guid) {
262 zcp->zc_present = B_TRUE;
263 zcp->zc_when = *loaded;
264 }
265 }
266
267 /*
268 * Iterate over all children.
269 */
270 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
271 &children) == 0) {
272 for (c = 0; c < children; c++)
273 zfs_mark_vdev(pool_guid, child[c], loaded);
274 }
275
276 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
277 &children) == 0) {
278 for (c = 0; c < children; c++)
279 zfs_mark_vdev(pool_guid, child[c], loaded);
280 }
281
282 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
283 &children) == 0) {
284 for (c = 0; c < children; c++)
285 zfs_mark_vdev(pool_guid, child[c], loaded);
286 }
287 }
288
289 static int
zfs_mark_pool(zpool_handle_t * zhp,void * unused)290 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
291 {
292 (void) unused;
293 zfs_case_t *zcp;
294 uint64_t pool_guid;
295 uint64_t *tod;
296 er_timeval_t loaded = { 0 };
297 nvlist_t *config, *vd;
298 uint_t nelem = 0;
299 int ret;
300
301 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
302 /*
303 * Mark any cases associated with just this pool.
304 */
305 for (zcp = list_head(&zfs_cases); zcp != NULL;
306 zcp = list_next(&zfs_cases, zcp)) {
307 if (zcp->zc_data.zc_pool_guid == pool_guid &&
308 zcp->zc_data.zc_vdev_guid == 0)
309 zcp->zc_present = B_TRUE;
310 }
311
312 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
313 zpool_close(zhp);
314 return (-1);
315 }
316
317 (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
318 &tod, &nelem);
319 if (nelem == 2) {
320 loaded.ertv_sec = tod[0];
321 loaded.ertv_nsec = tod[1];
322 for (zcp = list_head(&zfs_cases); zcp != NULL;
323 zcp = list_next(&zfs_cases, zcp)) {
324 if (zcp->zc_data.zc_pool_guid == pool_guid &&
325 zcp->zc_data.zc_vdev_guid == 0) {
326 zcp->zc_when = loaded;
327 }
328 }
329 }
330
331 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
332 if (ret) {
333 zpool_close(zhp);
334 return (-1);
335 }
336
337 zfs_mark_vdev(pool_guid, vd, &loaded);
338
339 zpool_close(zhp);
340
341 return (0);
342 }
343
344 struct load_time_arg {
345 uint64_t lt_guid;
346 er_timeval_t *lt_time;
347 boolean_t lt_found;
348 };
349
350 static int
zpool_find_load_time(zpool_handle_t * zhp,void * arg)351 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
352 {
353 struct load_time_arg *lta = arg;
354 uint64_t pool_guid;
355 uint64_t *tod;
356 nvlist_t *config;
357 uint_t nelem;
358
359 if (lta->lt_found) {
360 zpool_close(zhp);
361 return (0);
362 }
363
364 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
365 if (pool_guid != lta->lt_guid) {
366 zpool_close(zhp);
367 return (0);
368 }
369
370 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
371 zpool_close(zhp);
372 return (-1);
373 }
374
375 if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
376 &tod, &nelem) == 0 && nelem == 2) {
377 lta->lt_found = B_TRUE;
378 lta->lt_time->ertv_sec = tod[0];
379 lta->lt_time->ertv_nsec = tod[1];
380 }
381
382 zpool_close(zhp);
383
384 return (0);
385 }
386
387 static void
zfs_purge_cases(fmd_hdl_t * hdl)388 zfs_purge_cases(fmd_hdl_t *hdl)
389 {
390 zfs_case_t *zcp, *next;
391 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
392
393 /*
394 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No
395 * matter what we do, we're going to have to stomach an O(vdevs * cases)
396 * algorithm. In reality, both quantities are likely so small that
397 * neither will matter. Given that iterating over pools is more
398 * expensive than iterating over the in-memory case list, we opt for a
399 * 'present' flag in each case that starts off cleared. We then iterate
400 * over all pools, marking those that are still present, and removing
401 * those that aren't found.
402 *
403 * Note that we could also construct an FMRI and rely on
404 * fmd_nvl_fmri_present(), but this would end up doing the same search.
405 */
406
407 /*
408 * Mark the cases as not present.
409 */
410 for (zcp = list_head(&zfs_cases); zcp != NULL;
411 zcp = list_next(&zfs_cases, zcp))
412 zcp->zc_present = B_FALSE;
413
414 /*
415 * Iterate over all pools and mark the pools and vdevs found. If this
416 * fails (most probably because we're out of memory), then don't close
417 * any of the cases and we cannot be sure they are accurate.
418 */
419 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
420 return;
421
422 /*
423 * Remove those cases which were not found.
424 */
425 for (zcp = list_head(&zfs_cases); zcp != NULL; zcp = next) {
426 next = list_next(&zfs_cases, zcp);
427 if (!zcp->zc_present)
428 fmd_case_close(hdl, zcp->zc_case);
429 }
430 }
431
432 /*
433 * Construct the name of a serd engine given the pool/vdev GUID and type (io or
434 * checksum).
435 */
436 static void
zfs_serd_name(char * buf,uint64_t pool_guid,uint64_t vdev_guid,const char * type)437 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
438 const char *type)
439 {
440 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s",
441 (long long unsigned int)pool_guid,
442 (long long unsigned int)vdev_guid, type);
443 }
444
445 static void
zfs_case_retire(fmd_hdl_t * hdl,zfs_case_t * zcp)446 zfs_case_retire(fmd_hdl_t *hdl, zfs_case_t *zcp)
447 {
448 fmd_hdl_debug(hdl, "retiring case");
449
450 fmd_case_close(hdl, zcp->zc_case);
451 }
452
453 /*
454 * Solve a given ZFS case. This first checks to make sure the diagnosis is
455 * still valid, as well as cleaning up any pending timer associated with the
456 * case.
457 */
458 static void
zfs_case_solve(fmd_hdl_t * hdl,zfs_case_t * zcp,const char * faultname)459 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname)
460 {
461 nvlist_t *detector, *fault;
462 boolean_t serialize;
463 nvlist_t *fru = NULL;
464 fmd_hdl_debug(hdl, "solving fault '%s'", faultname);
465
466 /*
467 * Construct the detector from the case data. The detector is in the
468 * ZFS scheme, and is either the pool or the vdev, depending on whether
469 * this is a vdev or pool fault.
470 */
471 detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
472
473 (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
474 (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
475 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
476 zcp->zc_data.zc_pool_guid);
477 if (zcp->zc_data.zc_vdev_guid != 0) {
478 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
479 zcp->zc_data.zc_vdev_guid);
480 }
481
482 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
483 fru, detector);
484 fmd_case_add_suspect(hdl, zcp->zc_case, fault);
485
486 nvlist_free(fru);
487
488 fmd_case_solve(hdl, zcp->zc_case);
489
490 serialize = B_FALSE;
491 if (zcp->zc_data.zc_has_remove_timer) {
492 fmd_timer_remove(hdl, zcp->zc_remove_timer);
493 zcp->zc_data.zc_has_remove_timer = 0;
494 serialize = B_TRUE;
495 }
496 if (serialize)
497 zfs_case_serialize(zcp);
498
499 nvlist_free(detector);
500 }
501
502 static boolean_t
timeval_earlier(er_timeval_t * a,er_timeval_t * b)503 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
504 {
505 return (a->ertv_sec < b->ertv_sec ||
506 (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
507 }
508
509 static void
zfs_ereport_when(fmd_hdl_t * hdl,nvlist_t * nvl,er_timeval_t * when)510 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
511 {
512 (void) hdl;
513 int64_t *tod;
514 uint_t nelem;
515
516 if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod,
517 &nelem) == 0 && nelem == 2) {
518 when->ertv_sec = tod[0];
519 when->ertv_nsec = tod[1];
520 } else {
521 when->ertv_sec = when->ertv_nsec = UINT64_MAX;
522 }
523 }
524
525 /*
526 * Record the specified event in the SERD engine and return a
527 * boolean value indicating whether or not the engine fired as
528 * the result of inserting this event.
529 *
530 * When the pool has similar active cases on other vdevs, then
531 * the fired state is disregarded and the case is retired.
532 */
533 static int
zfs_fm_serd_record(fmd_hdl_t * hdl,const char * name,fmd_event_t * ep,zfs_case_t * zcp,const char * err_type)534 zfs_fm_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep,
535 zfs_case_t *zcp, const char *err_type)
536 {
537 int fired = fmd_serd_record(hdl, name, ep);
538 int peers = 0;
539
540 if (fired && (peers = zfs_other_serd_cases(hdl, &zcp->zc_data)) > 0) {
541 fmd_hdl_debug(hdl, "pool %llu is tracking %d other %s cases "
542 "-- skip faulting the vdev %llu",
543 (u_longlong_t)zcp->zc_data.zc_pool_guid,
544 peers, err_type,
545 (u_longlong_t)zcp->zc_data.zc_vdev_guid);
546 zfs_case_retire(hdl, zcp);
547 fired = 0;
548 }
549
550 return (fired);
551 }
552
553 /*
554 * Main fmd entry point.
555 */
556 static void
zfs_fm_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)557 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
558 {
559 zfs_case_t *zcp, *dcp;
560 int32_t pool_state;
561 uint64_t ena, pool_guid, vdev_guid, parent_guid;
562 uint64_t checksum_n, checksum_t;
563 uint64_t io_n, io_t;
564 er_timeval_t pool_load;
565 er_timeval_t er_when;
566 nvlist_t *detector;
567 boolean_t pool_found = B_FALSE;
568 boolean_t isresource;
569 const char *type;
570
571 /*
572 * We subscribe to notifications for vdev or pool removal. In these
573 * cases, there may be cases that no longer apply. Purge any cases
574 * that no longer apply.
575 */
576 if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) {
577 fmd_hdl_debug(hdl, "purging orphaned cases from %s",
578 strrchr(class, '.') + 1);
579 zfs_purge_cases(hdl);
580 zfs_stats.resource_drops.fmds_value.ui64++;
581 return;
582 }
583
584 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
585
586 if (isresource) {
587 /*
588 * For resources, we don't have a normal payload.
589 */
590 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
591 &vdev_guid) != 0)
592 pool_state = SPA_LOAD_OPEN;
593 else
594 pool_state = SPA_LOAD_NONE;
595 detector = NULL;
596 } else {
597 (void) nvlist_lookup_nvlist(nvl,
598 FM_EREPORT_DETECTOR, &detector);
599 (void) nvlist_lookup_int32(nvl,
600 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
601 }
602
603 /*
604 * We also ignore all ereports generated during an import of a pool,
605 * since the only possible fault (.pool) would result in import failure,
606 * and hence no persistent fault. Some day we may want to do something
607 * with these ereports, so we continue generating them internally.
608 */
609 if (pool_state == SPA_LOAD_IMPORT) {
610 zfs_stats.import_drops.fmds_value.ui64++;
611 fmd_hdl_debug(hdl, "ignoring '%s' during import", class);
612 return;
613 }
614
615 /*
616 * Device I/O errors are ignored during pool open.
617 */
618 if (pool_state == SPA_LOAD_OPEN &&
619 (fmd_nvl_class_match(hdl, nvl,
620 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
621 fmd_nvl_class_match(hdl, nvl,
622 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
623 fmd_nvl_class_match(hdl, nvl,
624 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
625 fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class);
626 zfs_stats.dev_drops.fmds_value.ui64++;
627 return;
628 }
629
630 /*
631 * We ignore ereports for anything except disks and files.
632 */
633 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
634 &type) == 0) {
635 if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
636 strcmp(type, VDEV_TYPE_FILE) != 0) {
637 zfs_stats.vdev_drops.fmds_value.ui64++;
638 return;
639 }
640 }
641
642 /*
643 * Determine if this ereport corresponds to an open case.
644 * Each vdev or pool can have a single case.
645 */
646 (void) nvlist_lookup_uint64(nvl,
647 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
648 if (nvlist_lookup_uint64(nvl,
649 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
650 vdev_guid = 0;
651 if (nvlist_lookup_uint64(nvl,
652 FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID, &parent_guid) != 0)
653 parent_guid = 0;
654 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
655 ena = 0;
656
657 zfs_ereport_when(hdl, nvl, &er_when);
658
659 for (zcp = list_head(&zfs_cases); zcp != NULL;
660 zcp = list_next(&zfs_cases, zcp)) {
661 if (zcp->zc_data.zc_pool_guid == pool_guid) {
662 pool_found = B_TRUE;
663 pool_load = zcp->zc_when;
664 }
665 if (zcp->zc_data.zc_vdev_guid == vdev_guid)
666 break;
667 }
668
669 /*
670 * Avoid falsely accusing a pool of being faulty. Do so by
671 * not replaying ereports that were generated prior to the
672 * current import. If the failure that generated them was
673 * transient because the device was actually removed but we
674 * didn't receive the normal asynchronous notification, we
675 * don't want to mark it as faulted and potentially panic. If
676 * there is still a problem we'd expect not to be able to
677 * import the pool, or that new ereports will be generated
678 * once the pool is used.
679 */
680 if (pool_found && timeval_earlier(&er_when, &pool_load)) {
681 fmd_hdl_debug(hdl, "ignoring pool %llx, "
682 "ereport time %lld.%lld, pool load time = %lld.%lld",
683 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
684 pool_load.ertv_sec, pool_load.ertv_nsec);
685 zfs_stats.old_drops.fmds_value.ui64++;
686 return;
687 }
688
689 if (!pool_found) {
690 /*
691 * Haven't yet seen this pool, but same situation
692 * may apply.
693 */
694 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
695 struct load_time_arg la;
696
697 la.lt_guid = pool_guid;
698 la.lt_time = &pool_load;
699 la.lt_found = B_FALSE;
700
701 if (zhdl != NULL &&
702 zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
703 la.lt_found == B_TRUE) {
704 pool_found = B_TRUE;
705
706 if (timeval_earlier(&er_when, &pool_load)) {
707 fmd_hdl_debug(hdl, "ignoring pool %llx, "
708 "ereport time %lld.%lld, "
709 "pool load time = %lld.%lld",
710 pool_guid, er_when.ertv_sec,
711 er_when.ertv_nsec, pool_load.ertv_sec,
712 pool_load.ertv_nsec);
713 zfs_stats.old_drops.fmds_value.ui64++;
714 return;
715 }
716 }
717 }
718
719 if (zcp == NULL) {
720 fmd_case_t *cs;
721 zfs_case_data_t data = { 0 };
722
723 /*
724 * If this is one of our 'fake' resource ereports, and there is
725 * no case open, simply discard it.
726 */
727 if (isresource) {
728 zfs_stats.resource_drops.fmds_value.ui64++;
729 fmd_hdl_debug(hdl, "discarding '%s for vdev %llu",
730 class, vdev_guid);
731 return;
732 }
733
734 /*
735 * Skip tracking some ereports
736 */
737 if (strcmp(class,
738 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 ||
739 strcmp(class,
740 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0) {
741 zfs_stats.resource_drops.fmds_value.ui64++;
742 return;
743 }
744
745 /*
746 * Open a new case.
747 */
748 cs = fmd_case_open(hdl, NULL);
749
750 fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'",
751 vdev_guid, class);
752
753 /*
754 * Initialize the case buffer. To commonize code, we actually
755 * create the buffer with existing data, and then call
756 * zfs_case_unserialize() to instantiate the in-core structure.
757 */
758 fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t));
759
760 data.zc_version = CASE_DATA_VERSION_SERD;
761 data.zc_ena = ena;
762 data.zc_pool_guid = pool_guid;
763 data.zc_vdev_guid = vdev_guid;
764 data.zc_parent_guid = parent_guid;
765 data.zc_pool_state = (int)pool_state;
766
767 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
768
769 zcp = zfs_case_unserialize(hdl, cs);
770 assert(zcp != NULL);
771 if (pool_found)
772 zcp->zc_when = pool_load;
773 }
774
775 if (isresource) {
776 fmd_hdl_debug(hdl, "resource event '%s'", class);
777
778 if (fmd_nvl_class_match(hdl, nvl,
779 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
780 /*
781 * The 'resource.fs.zfs.autoreplace' event indicates
782 * that the pool was loaded with the 'autoreplace'
783 * property set. In this case, any pending device
784 * failures should be ignored, as the asynchronous
785 * autoreplace handling will take care of them.
786 */
787 fmd_case_close(hdl, zcp->zc_case);
788 } else if (fmd_nvl_class_match(hdl, nvl,
789 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
790 /*
791 * The 'resource.fs.zfs.removed' event indicates that
792 * device removal was detected, and the device was
793 * closed asynchronously. If this is the case, we
794 * assume that any recent I/O errors were due to the
795 * device removal, not any fault of the device itself.
796 * We reset the SERD engine, and cancel any pending
797 * timers.
798 */
799 if (zcp->zc_data.zc_has_remove_timer) {
800 fmd_timer_remove(hdl, zcp->zc_remove_timer);
801 zcp->zc_data.zc_has_remove_timer = 0;
802 zfs_case_serialize(zcp);
803 }
804 if (zcp->zc_data.zc_serd_io[0] != '\0')
805 fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io);
806 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
807 fmd_serd_reset(hdl,
808 zcp->zc_data.zc_serd_checksum);
809 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
810 fmd_serd_reset(hdl,
811 zcp->zc_data.zc_serd_slow_io);
812 } else if (fmd_nvl_class_match(hdl, nvl,
813 ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) {
814 uint64_t state = 0;
815
816 if (zcp != NULL &&
817 nvlist_lookup_uint64(nvl,
818 FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 &&
819 state == VDEV_STATE_HEALTHY) {
820 fmd_hdl_debug(hdl, "closing case after a "
821 "device statechange to healthy");
822 fmd_case_close(hdl, zcp->zc_case);
823 }
824 }
825 zfs_stats.resource_drops.fmds_value.ui64++;
826 return;
827 }
828
829 /*
830 * Associate the ereport with this case.
831 */
832 fmd_case_add_ereport(hdl, zcp->zc_case, ep);
833
834 /*
835 * Don't do anything else if this case is already solved.
836 */
837 if (fmd_case_solved(hdl, zcp->zc_case))
838 return;
839
840 if (vdev_guid)
841 fmd_hdl_debug(hdl, "error event '%s', vdev %llu", class,
842 vdev_guid);
843 else
844 fmd_hdl_debug(hdl, "error event '%s'", class);
845
846 /*
847 * Determine if we should solve the case and generate a fault. We solve
848 * a case if:
849 *
850 * a. A pool failed to open (ereport.fs.zfs.pool)
851 * b. A device failed to open (ereport.fs.zfs.pool) while a pool
852 * was up and running.
853 *
854 * We may see a series of ereports associated with a pool open, all
855 * chained together by the same ENA. If the pool open succeeds, then
856 * we'll see no further ereports. To detect when a pool open has
857 * succeeded, we associate a timer with the event. When it expires, we
858 * close the case.
859 */
860 if (fmd_nvl_class_match(hdl, nvl,
861 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
862 /*
863 * Pool level fault. Before solving the case, go through and
864 * close any open device cases that may be pending.
865 */
866 for (dcp = list_head(&zfs_cases); dcp != NULL;
867 dcp = list_next(&zfs_cases, dcp)) {
868 if (dcp->zc_data.zc_pool_guid ==
869 zcp->zc_data.zc_pool_guid &&
870 dcp->zc_data.zc_vdev_guid != 0)
871 fmd_case_close(hdl, dcp->zc_case);
872 }
873
874 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool");
875 } else if (fmd_nvl_class_match(hdl, nvl,
876 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
877 /*
878 * Pool level fault for reading the intent logs.
879 */
880 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay");
881 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
882 /*
883 * Device fault.
884 */
885 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device");
886 } else if (fmd_nvl_class_match(hdl, nvl,
887 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
888 fmd_nvl_class_match(hdl, nvl,
889 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
890 fmd_nvl_class_match(hdl, nvl,
891 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
892 fmd_nvl_class_match(hdl, nvl,
893 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) ||
894 fmd_nvl_class_match(hdl, nvl,
895 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
896 const char *failmode = NULL;
897 boolean_t checkremove = B_FALSE;
898 uint32_t pri = 0;
899
900 /*
901 * If this is a checksum or I/O error, then toss it into the
902 * appropriate SERD engine and check to see if it has fired.
903 * Ideally, we want to do something more sophisticated,
904 * (persistent errors for a single data block, etc). For now,
905 * a single SERD engine is sufficient.
906 */
907 if (fmd_nvl_class_match(hdl, nvl,
908 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
909 if (zcp->zc_data.zc_serd_io[0] == '\0') {
910 if (nvlist_lookup_uint64(nvl,
911 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
912 &io_n) != 0) {
913 io_n = DEFAULT_IO_N;
914 }
915 if (nvlist_lookup_uint64(nvl,
916 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
917 &io_t) != 0) {
918 io_t = DEFAULT_IO_T;
919 }
920 zfs_serd_name(zcp->zc_data.zc_serd_io,
921 pool_guid, vdev_guid, "io");
922 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
923 io_n,
924 SEC2NSEC(io_t));
925 zfs_case_serialize(zcp);
926 }
927 if (zfs_fm_serd_record(hdl, zcp->zc_data.zc_serd_io,
928 ep, zcp, "io error")) {
929 checkremove = B_TRUE;
930 }
931 } else if (fmd_nvl_class_match(hdl, nvl,
932 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY))) {
933 uint64_t slow_io_n, slow_io_t;
934
935 /*
936 * Create a slow io SERD engine when the VDEV has the
937 * 'vdev_slow_io_n' and 'vdev_slow_io_n' properties.
938 */
939 if (zcp->zc_data.zc_serd_slow_io[0] == '\0' &&
940 nvlist_lookup_uint64(nvl,
941 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N,
942 &slow_io_n) == 0 &&
943 nvlist_lookup_uint64(nvl,
944 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T,
945 &slow_io_t) == 0) {
946 zfs_serd_name(zcp->zc_data.zc_serd_slow_io,
947 pool_guid, vdev_guid, "slow_io");
948 fmd_serd_create(hdl,
949 zcp->zc_data.zc_serd_slow_io,
950 slow_io_n,
951 SEC2NSEC(slow_io_t));
952 zfs_case_serialize(zcp);
953 }
954 /* Pass event to SERD engine and see if this triggers */
955 if (zcp->zc_data.zc_serd_slow_io[0] != '\0' &&
956 zfs_fm_serd_record(hdl,
957 zcp->zc_data.zc_serd_slow_io, ep, zcp, "slow io")) {
958 zfs_case_solve(hdl, zcp,
959 "fault.fs.zfs.vdev.slow_io");
960 }
961 } else if (fmd_nvl_class_match(hdl, nvl,
962 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
963 uint64_t flags = 0;
964 int32_t flags32 = 0;
965 /*
966 * We ignore ereports for checksum errors generated by
967 * scrub/resilver I/O to avoid potentially further
968 * degrading the pool while it's being repaired.
969 *
970 * Note that FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS used to
971 * be int32. To allow newer zed to work on older
972 * kernels, if we don't find the flags, we look for
973 * the older ones too.
974 */
975 if (((nvlist_lookup_uint32(nvl,
976 FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY, &pri) == 0) &&
977 (pri == ZIO_PRIORITY_SCRUB ||
978 pri == ZIO_PRIORITY_REBUILD)) ||
979 ((nvlist_lookup_uint64(nvl,
980 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags) == 0) &&
981 (flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) ||
982 ((nvlist_lookup_int32(nvl,
983 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags32) == 0) &&
984 (flags32 & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)))) {
985 fmd_hdl_debug(hdl, "ignoring '%s' for "
986 "scrub/resilver I/O", class);
987 return;
988 }
989
990 if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
991 if (nvlist_lookup_uint64(nvl,
992 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
993 &checksum_n) != 0) {
994 checksum_n = DEFAULT_CHECKSUM_N;
995 }
996 if (nvlist_lookup_uint64(nvl,
997 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
998 &checksum_t) != 0) {
999 checksum_t = DEFAULT_CHECKSUM_T;
1000 }
1001
1002 zfs_serd_name(zcp->zc_data.zc_serd_checksum,
1003 pool_guid, vdev_guid, "checksum");
1004 fmd_serd_create(hdl,
1005 zcp->zc_data.zc_serd_checksum,
1006 checksum_n,
1007 SEC2NSEC(checksum_t));
1008 zfs_case_serialize(zcp);
1009 }
1010 if (zfs_fm_serd_record(hdl,
1011 zcp->zc_data.zc_serd_checksum, ep, zcp,
1012 "checksum")) {
1013 zfs_case_solve(hdl, zcp,
1014 "fault.fs.zfs.vdev.checksum");
1015 }
1016 } else if (fmd_nvl_class_match(hdl, nvl,
1017 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
1018 (nvlist_lookup_string(nvl,
1019 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
1020 failmode != NULL) {
1021 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
1022 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
1023 zfs_case_solve(hdl, zcp,
1024 "fault.fs.zfs.io_failure_continue");
1025 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
1026 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
1027 zfs_case_solve(hdl, zcp,
1028 "fault.fs.zfs.io_failure_wait");
1029 }
1030 } else if (fmd_nvl_class_match(hdl, nvl,
1031 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
1032 #ifndef __linux__
1033 /* This causes an unexpected fault diagnosis on linux */
1034 checkremove = B_TRUE;
1035 #endif
1036 }
1037
1038 /*
1039 * Because I/O errors may be due to device removal, we postpone
1040 * any diagnosis until we're sure that we aren't about to
1041 * receive a 'resource.fs.zfs.removed' event.
1042 */
1043 if (checkremove) {
1044 if (zcp->zc_data.zc_has_remove_timer)
1045 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1046 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
1047 zfs_remove_timeout);
1048 if (!zcp->zc_data.zc_has_remove_timer) {
1049 zcp->zc_data.zc_has_remove_timer = 1;
1050 zfs_case_serialize(zcp);
1051 }
1052 }
1053 }
1054 }
1055
1056 /*
1057 * The timeout is fired when we diagnosed an I/O error, and it was not due to
1058 * device removal (which would cause the timeout to be cancelled).
1059 */
1060 static void
zfs_fm_timeout(fmd_hdl_t * hdl,id_t id,void * data)1061 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
1062 {
1063 zfs_case_t *zcp = data;
1064
1065 if (id == zcp->zc_remove_timer)
1066 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io");
1067 }
1068
1069 /*
1070 * The specified case has been closed and any case-specific
1071 * data structures should be deallocated.
1072 */
1073 static void
zfs_fm_close(fmd_hdl_t * hdl,fmd_case_t * cs)1074 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
1075 {
1076 zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
1077
1078 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
1079 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
1080 if (zcp->zc_data.zc_serd_io[0] != '\0')
1081 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
1082 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
1083 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_slow_io);
1084 if (zcp->zc_data.zc_has_remove_timer)
1085 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1086
1087 list_remove(&zfs_cases, zcp);
1088 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1089 }
1090
1091 static const fmd_hdl_ops_t fmd_ops = {
1092 zfs_fm_recv, /* fmdo_recv */
1093 zfs_fm_timeout, /* fmdo_timeout */
1094 zfs_fm_close, /* fmdo_close */
1095 NULL, /* fmdo_stats */
1096 NULL, /* fmdo_gc */
1097 };
1098
1099 static const fmd_prop_t fmd_props[] = {
1100 { NULL, 0, NULL }
1101 };
1102
1103 static const fmd_hdl_info_t fmd_info = {
1104 "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
1105 };
1106
1107 void
_zfs_diagnosis_init(fmd_hdl_t * hdl)1108 _zfs_diagnosis_init(fmd_hdl_t *hdl)
1109 {
1110 libzfs_handle_t *zhdl;
1111
1112 if ((zhdl = libzfs_init()) == NULL)
1113 return;
1114
1115 list_create(&zfs_cases,
1116 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node));
1117
1118 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
1119 list_destroy(&zfs_cases);
1120 libzfs_fini(zhdl);
1121 return;
1122 }
1123
1124 fmd_hdl_setspecific(hdl, zhdl);
1125
1126 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
1127 sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
1128 }
1129
1130 void
_zfs_diagnosis_fini(fmd_hdl_t * hdl)1131 _zfs_diagnosis_fini(fmd_hdl_t *hdl)
1132 {
1133 zfs_case_t *zcp;
1134 libzfs_handle_t *zhdl;
1135
1136 /*
1137 * Remove all active cases.
1138 */
1139 while ((zcp = list_remove_head(&zfs_cases)) != NULL) {
1140 fmd_hdl_debug(hdl, "removing case ena %llu",
1141 (long long unsigned)zcp->zc_data.zc_ena);
1142 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1143 }
1144
1145 list_destroy(&zfs_cases);
1146
1147 zhdl = fmd_hdl_getspecific(hdl);
1148 libzfs_fini(zhdl);
1149 }
1150