1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
15 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
16 * Copyright (c) 2016, Intel Corporation.
17 * Copyright (c) 2023, Klara Inc.
18 */
19
20 #include <stddef.h>
21 #include <string.h>
22 #include <libzfs.h>
23 #include <sys/types.h>
24 #include <sys/time.h>
25 #include <sys/fs/zfs.h>
26 #include <sys/fm/protocol.h>
27 #include <sys/fm/fs/zfs.h>
28 #include <sys/zio.h>
29
30 #include "zfs_agents.h"
31 #include "fmd_api.h"
32
33 /*
34 * Default values for the serd engine when processing checksum or io errors. The
35 * semantics are N <events> in T <seconds>.
36 */
37 #define DEFAULT_CHECKSUM_N 10 /* events */
38 #define DEFAULT_CHECKSUM_T 600 /* seconds */
39 #define DEFAULT_IO_N 10 /* events */
40 #define DEFAULT_IO_T 600 /* seconds */
41
42 #define CASE_GC_TIMEOUT_SECS 43200 /* 12 hours */
43
44 /*
45 * Our serd engines are named in the following format:
46 * 'zfs_<pool_guid>_<vdev_guid>_{checksum,io,slow_io}'
47 * This #define reserves enough space for two 64-bit hex values plus the
48 * length of the longest string.
49 */
50 #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))
51
52 /*
53 * On-disk case structure. This must maintain backwards compatibility with
54 * previous versions of the DE. By default, any members appended to the end
55 * will be filled with zeros if they don't exist in a previous version.
56 */
57 typedef struct zfs_case_data {
58 uint64_t zc_version;
59 uint64_t zc_ena;
60 uint64_t zc_pool_guid;
61 uint64_t zc_vdev_guid;
62 uint64_t zc_parent_guid;
63 int zc_pool_state;
64 char zc_serd_checksum[MAX_SERDLEN];
65 char zc_serd_io[MAX_SERDLEN];
66 char zc_serd_slow_io[MAX_SERDLEN];
67 int zc_has_remove_timer;
68 } zfs_case_data_t;
69
70 /*
71 * Time-of-day
72 */
73 typedef struct er_timeval {
74 uint64_t ertv_sec;
75 uint64_t ertv_nsec;
76 } er_timeval_t;
77
78 /*
79 * In-core case structure.
80 */
81 typedef struct zfs_case {
82 boolean_t zc_present;
83 uint32_t zc_version;
84 zfs_case_data_t zc_data;
85 fmd_case_t *zc_case;
86 list_node_t zc_node;
87 id_t zc_remove_timer;
88 char *zc_fru;
89 er_timeval_t zc_when;
90 } zfs_case_t;
91
92 #define CASE_DATA "data"
93 #define CASE_FRU "fru"
94 #define CASE_DATA_VERSION_INITIAL 1
95 #define CASE_DATA_VERSION_SERD 2
96
97 typedef struct zfs_de_stats {
98 fmd_stat_t old_drops;
99 fmd_stat_t dev_drops;
100 fmd_stat_t vdev_drops;
101 fmd_stat_t import_drops;
102 fmd_stat_t resource_drops;
103 } zfs_de_stats_t;
104
105 zfs_de_stats_t zfs_stats = {
106 { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
107 { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
108 { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
109 { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
110 { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
111 };
112
113 /* wait 15 seconds after a removal */
114 static hrtime_t zfs_remove_timeout = SEC2NSEC(15);
115
116 static list_t zfs_cases;
117
118 #define ZFS_MAKE_RSRC(type) \
119 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
120 #define ZFS_MAKE_EREPORT(type) \
121 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
122
123 static void zfs_purge_cases(fmd_hdl_t *hdl);
124
125 /*
126 * Write out the persistent representation of an active case.
127 */
128 static void
zfs_case_serialize(zfs_case_t * zcp)129 zfs_case_serialize(zfs_case_t *zcp)
130 {
131 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
132 }
133
134 /*
135 * Read back the persistent representation of an active case.
136 */
137 static zfs_case_t *
zfs_case_unserialize(fmd_hdl_t * hdl,fmd_case_t * cp)138 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
139 {
140 zfs_case_t *zcp;
141
142 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
143 zcp->zc_case = cp;
144
145 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
146 sizeof (zcp->zc_data));
147
148 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
149 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
150 return (NULL);
151 }
152
153 /*
154 * fmd_buf_read() will have already zeroed out the remainder of the
155 * buffer, so we don't have to do anything special if the version
156 * doesn't include the SERD engine name.
157 */
158
159 if (zcp->zc_data.zc_has_remove_timer)
160 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
161 NULL, zfs_remove_timeout);
162
163 list_link_init(&zcp->zc_node);
164 list_insert_head(&zfs_cases, zcp);
165
166 fmd_case_setspecific(hdl, cp, zcp);
167
168 return (zcp);
169 }
170
171 /*
172 * Return count of other unique SERD cases under same vdev parent
173 */
174 static uint_t
zfs_other_serd_cases(fmd_hdl_t * hdl,const zfs_case_data_t * zfs_case)175 zfs_other_serd_cases(fmd_hdl_t *hdl, const zfs_case_data_t *zfs_case)
176 {
177 zfs_case_t *zcp;
178 uint_t cases = 0;
179 static hrtime_t next_check = 0;
180
181 /*
182 * Note that plumbing in some external GC would require adding locking,
183 * since most of this module code is not thread safe and assumes there
184 * is only one thread running against the module. So we perform GC here
185 * inline periodically so that future delay induced faults will be
186 * possible once the issue causing multiple vdev delays is resolved.
187 */
188 if (gethrestime_sec() > next_check) {
189 /* Periodically purge old SERD entries and stale cases */
190 fmd_serd_gc(hdl);
191 zfs_purge_cases(hdl);
192 next_check = gethrestime_sec() + CASE_GC_TIMEOUT_SECS;
193 }
194
195 for (zcp = list_head(&zfs_cases); zcp != NULL;
196 zcp = list_next(&zfs_cases, zcp)) {
197 zfs_case_data_t *zcd = &zcp->zc_data;
198
199 /*
200 * must be same pool and parent vdev but different leaf vdev
201 */
202 if (zcd->zc_pool_guid != zfs_case->zc_pool_guid ||
203 zcd->zc_parent_guid != zfs_case->zc_parent_guid ||
204 zcd->zc_vdev_guid == zfs_case->zc_vdev_guid) {
205 continue;
206 }
207
208 /*
209 * Check if there is another active serd case besides zfs_case
210 *
211 * Only one serd engine will be assigned to the case
212 */
213 if (zcd->zc_serd_checksum[0] == zfs_case->zc_serd_checksum[0] &&
214 fmd_serd_active(hdl, zcd->zc_serd_checksum)) {
215 cases++;
216 }
217 if (zcd->zc_serd_io[0] == zfs_case->zc_serd_io[0] &&
218 fmd_serd_active(hdl, zcd->zc_serd_io)) {
219 cases++;
220 }
221 if (zcd->zc_serd_slow_io[0] == zfs_case->zc_serd_slow_io[0] &&
222 fmd_serd_active(hdl, zcd->zc_serd_slow_io)) {
223 cases++;
224 }
225 }
226 return (cases);
227 }
228
229 /*
230 * Iterate over any active cases. If any cases are associated with a pool or
231 * vdev which is no longer present on the system, close the associated case.
232 */
233 static void
zfs_mark_vdev(uint64_t pool_guid,nvlist_t * vd,er_timeval_t * loaded)234 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
235 {
236 uint64_t vdev_guid = 0;
237 uint_t c, children;
238 nvlist_t **child;
239 zfs_case_t *zcp;
240
241 (void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
242
243 /*
244 * Mark any cases associated with this (pool, vdev) pair.
245 */
246 for (zcp = list_head(&zfs_cases); zcp != NULL;
247 zcp = list_next(&zfs_cases, zcp)) {
248 if (zcp->zc_data.zc_pool_guid == pool_guid &&
249 zcp->zc_data.zc_vdev_guid == vdev_guid) {
250 zcp->zc_present = B_TRUE;
251 zcp->zc_when = *loaded;
252 }
253 }
254
255 /*
256 * Iterate over all children.
257 */
258 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
259 &children) == 0) {
260 for (c = 0; c < children; c++)
261 zfs_mark_vdev(pool_guid, child[c], loaded);
262 }
263
264 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
265 &children) == 0) {
266 for (c = 0; c < children; c++)
267 zfs_mark_vdev(pool_guid, child[c], loaded);
268 }
269
270 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
271 &children) == 0) {
272 for (c = 0; c < children; c++)
273 zfs_mark_vdev(pool_guid, child[c], loaded);
274 }
275 }
276
277 static int
zfs_mark_pool(zpool_handle_t * zhp,void * unused)278 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
279 {
280 (void) unused;
281 zfs_case_t *zcp;
282 uint64_t pool_guid;
283 uint64_t *tod;
284 er_timeval_t loaded = { 0 };
285 nvlist_t *config, *vd;
286 uint_t nelem = 0;
287 int ret;
288
289 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
290 /*
291 * Mark any cases associated with just this pool.
292 */
293 for (zcp = list_head(&zfs_cases); zcp != NULL;
294 zcp = list_next(&zfs_cases, zcp)) {
295 if (zcp->zc_data.zc_pool_guid == pool_guid &&
296 zcp->zc_data.zc_vdev_guid == 0)
297 zcp->zc_present = B_TRUE;
298 }
299
300 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
301 zpool_close(zhp);
302 return (-1);
303 }
304
305 (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
306 &tod, &nelem);
307 if (nelem == 2) {
308 loaded.ertv_sec = tod[0];
309 loaded.ertv_nsec = tod[1];
310 for (zcp = list_head(&zfs_cases); zcp != NULL;
311 zcp = list_next(&zfs_cases, zcp)) {
312 if (zcp->zc_data.zc_pool_guid == pool_guid &&
313 zcp->zc_data.zc_vdev_guid == 0) {
314 zcp->zc_when = loaded;
315 }
316 }
317 }
318
319 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
320 if (ret) {
321 zpool_close(zhp);
322 return (-1);
323 }
324
325 zfs_mark_vdev(pool_guid, vd, &loaded);
326
327 zpool_close(zhp);
328
329 return (0);
330 }
331
332 struct load_time_arg {
333 uint64_t lt_guid;
334 er_timeval_t *lt_time;
335 boolean_t lt_found;
336 };
337
338 static int
zpool_find_load_time(zpool_handle_t * zhp,void * arg)339 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
340 {
341 struct load_time_arg *lta = arg;
342 uint64_t pool_guid;
343 uint64_t *tod;
344 nvlist_t *config;
345 uint_t nelem;
346
347 if (lta->lt_found) {
348 zpool_close(zhp);
349 return (0);
350 }
351
352 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
353 if (pool_guid != lta->lt_guid) {
354 zpool_close(zhp);
355 return (0);
356 }
357
358 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
359 zpool_close(zhp);
360 return (-1);
361 }
362
363 if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
364 &tod, &nelem) == 0 && nelem == 2) {
365 lta->lt_found = B_TRUE;
366 lta->lt_time->ertv_sec = tod[0];
367 lta->lt_time->ertv_nsec = tod[1];
368 }
369
370 zpool_close(zhp);
371
372 return (0);
373 }
374
375 static void
zfs_purge_cases(fmd_hdl_t * hdl)376 zfs_purge_cases(fmd_hdl_t *hdl)
377 {
378 zfs_case_t *zcp, *next;
379 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
380
381 /*
382 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No
383 * matter what we do, we're going to have to stomach an O(vdevs * cases)
384 * algorithm. In reality, both quantities are likely so small that
385 * neither will matter. Given that iterating over pools is more
386 * expensive than iterating over the in-memory case list, we opt for a
387 * 'present' flag in each case that starts off cleared. We then iterate
388 * over all pools, marking those that are still present, and removing
389 * those that aren't found.
390 *
391 * Note that we could also construct an FMRI and rely on
392 * fmd_nvl_fmri_present(), but this would end up doing the same search.
393 */
394
395 /*
396 * Mark the cases as not present.
397 */
398 for (zcp = list_head(&zfs_cases); zcp != NULL;
399 zcp = list_next(&zfs_cases, zcp))
400 zcp->zc_present = B_FALSE;
401
402 /*
403 * Iterate over all pools and mark the pools and vdevs found. If this
404 * fails (most probably because we're out of memory), then don't close
405 * any of the cases and we cannot be sure they are accurate.
406 */
407 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
408 return;
409
410 /*
411 * Remove those cases which were not found.
412 */
413 for (zcp = list_head(&zfs_cases); zcp != NULL; zcp = next) {
414 next = list_next(&zfs_cases, zcp);
415 if (!zcp->zc_present)
416 fmd_case_close(hdl, zcp->zc_case);
417 }
418 }
419
420 /*
421 * Construct the name of a serd engine given the pool/vdev GUID and type (io or
422 * checksum).
423 */
424 static void
zfs_serd_name(char * buf,uint64_t pool_guid,uint64_t vdev_guid,const char * type)425 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
426 const char *type)
427 {
428 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s",
429 (long long unsigned int)pool_guid,
430 (long long unsigned int)vdev_guid, type);
431 }
432
433 static void
zfs_case_retire(fmd_hdl_t * hdl,zfs_case_t * zcp)434 zfs_case_retire(fmd_hdl_t *hdl, zfs_case_t *zcp)
435 {
436 fmd_hdl_debug(hdl, "retiring case");
437
438 fmd_case_close(hdl, zcp->zc_case);
439 }
440
441 /*
442 * Solve a given ZFS case. This first checks to make sure the diagnosis is
443 * still valid, as well as cleaning up any pending timer associated with the
444 * case.
445 */
446 static void
zfs_case_solve(fmd_hdl_t * hdl,zfs_case_t * zcp,const char * faultname)447 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname)
448 {
449 nvlist_t *detector, *fault;
450 boolean_t serialize;
451 nvlist_t *fru = NULL;
452 fmd_hdl_debug(hdl, "solving fault '%s'", faultname);
453
454 /*
455 * Construct the detector from the case data. The detector is in the
456 * ZFS scheme, and is either the pool or the vdev, depending on whether
457 * this is a vdev or pool fault.
458 */
459 detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
460
461 (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
462 (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
463 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
464 zcp->zc_data.zc_pool_guid);
465 if (zcp->zc_data.zc_vdev_guid != 0) {
466 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
467 zcp->zc_data.zc_vdev_guid);
468 }
469
470 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
471 fru, detector);
472 fmd_case_add_suspect(hdl, zcp->zc_case, fault);
473
474 nvlist_free(fru);
475
476 fmd_case_solve(hdl, zcp->zc_case);
477
478 serialize = B_FALSE;
479 if (zcp->zc_data.zc_has_remove_timer) {
480 fmd_timer_remove(hdl, zcp->zc_remove_timer);
481 zcp->zc_data.zc_has_remove_timer = 0;
482 serialize = B_TRUE;
483 }
484 if (serialize)
485 zfs_case_serialize(zcp);
486
487 nvlist_free(detector);
488 }
489
490 static boolean_t
timeval_earlier(er_timeval_t * a,er_timeval_t * b)491 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
492 {
493 return (a->ertv_sec < b->ertv_sec ||
494 (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
495 }
496
497 static void
zfs_ereport_when(fmd_hdl_t * hdl,nvlist_t * nvl,er_timeval_t * when)498 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
499 {
500 (void) hdl;
501 int64_t *tod;
502 uint_t nelem;
503
504 if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod,
505 &nelem) == 0 && nelem == 2) {
506 when->ertv_sec = tod[0];
507 when->ertv_nsec = tod[1];
508 } else {
509 when->ertv_sec = when->ertv_nsec = UINT64_MAX;
510 }
511 }
512
513 /*
514 * Record the specified event in the SERD engine and return a
515 * boolean value indicating whether or not the engine fired as
516 * the result of inserting this event.
517 *
518 * When the pool has similar active cases on other vdevs, then
519 * the fired state is disregarded and the case is retired.
520 */
521 static int
zfs_fm_serd_record(fmd_hdl_t * hdl,const char * name,fmd_event_t * ep,zfs_case_t * zcp,const char * err_type)522 zfs_fm_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep,
523 zfs_case_t *zcp, const char *err_type)
524 {
525 int fired = fmd_serd_record(hdl, name, ep);
526 int peers = 0;
527
528 if (fired && (peers = zfs_other_serd_cases(hdl, &zcp->zc_data)) > 0) {
529 fmd_hdl_debug(hdl, "pool %llu is tracking %d other %s cases "
530 "-- skip faulting the vdev %llu",
531 (u_longlong_t)zcp->zc_data.zc_pool_guid,
532 peers, err_type,
533 (u_longlong_t)zcp->zc_data.zc_vdev_guid);
534 zfs_case_retire(hdl, zcp);
535 fired = 0;
536 }
537
538 return (fired);
539 }
540
541 /*
542 * Main fmd entry point.
543 */
544 static void
zfs_fm_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)545 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
546 {
547 zfs_case_t *zcp, *dcp;
548 int32_t pool_state;
549 uint64_t ena, pool_guid, vdev_guid, parent_guid;
550 uint64_t checksum_n, checksum_t;
551 uint64_t io_n, io_t;
552 er_timeval_t pool_load = {0};
553 er_timeval_t er_when;
554 nvlist_t *detector;
555 boolean_t pool_found = B_FALSE;
556 boolean_t isresource;
557 const char *type;
558
559 /*
560 * We subscribe to notifications for vdev or pool removal. In these
561 * cases, there may be cases that no longer apply. Purge any cases
562 * that no longer apply.
563 */
564 if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) {
565 fmd_hdl_debug(hdl, "purging orphaned cases from %s",
566 strrchr(class, '.') + 1);
567 zfs_purge_cases(hdl);
568 zfs_stats.resource_drops.fmds_value.ui64++;
569 return;
570 }
571
572 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
573
574 if (isresource) {
575 /*
576 * For resources, we don't have a normal payload.
577 */
578 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
579 &vdev_guid) != 0)
580 pool_state = SPA_LOAD_OPEN;
581 else
582 pool_state = SPA_LOAD_NONE;
583 detector = NULL;
584 } else {
585 (void) nvlist_lookup_nvlist(nvl,
586 FM_EREPORT_DETECTOR, &detector);
587 (void) nvlist_lookup_int32(nvl,
588 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
589 }
590
591 /*
592 * We also ignore all ereports generated during an import of a pool,
593 * since the only possible fault (.pool) would result in import failure,
594 * and hence no persistent fault. Some day we may want to do something
595 * with these ereports, so we continue generating them internally.
596 */
597 if (pool_state == SPA_LOAD_IMPORT) {
598 zfs_stats.import_drops.fmds_value.ui64++;
599 fmd_hdl_debug(hdl, "ignoring '%s' during import", class);
600 return;
601 }
602
603 /*
604 * Device I/O errors are ignored during pool open.
605 */
606 if (pool_state == SPA_LOAD_OPEN &&
607 (fmd_nvl_class_match(hdl, nvl,
608 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
609 fmd_nvl_class_match(hdl, nvl,
610 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
611 fmd_nvl_class_match(hdl, nvl,
612 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
613 fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class);
614 zfs_stats.dev_drops.fmds_value.ui64++;
615 return;
616 }
617
618 /*
619 * We ignore ereports for anything except disks and files.
620 */
621 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
622 &type) == 0) {
623 if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
624 strcmp(type, VDEV_TYPE_FILE) != 0) {
625 zfs_stats.vdev_drops.fmds_value.ui64++;
626 return;
627 }
628 }
629
630 /*
631 * Determine if this ereport corresponds to an open case.
632 * Each vdev or pool can have a single case.
633 */
634 (void) nvlist_lookup_uint64(nvl,
635 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
636 if (nvlist_lookup_uint64(nvl,
637 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
638 vdev_guid = 0;
639 if (nvlist_lookup_uint64(nvl,
640 FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID, &parent_guid) != 0)
641 parent_guid = 0;
642 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
643 ena = 0;
644
645 zfs_ereport_when(hdl, nvl, &er_when);
646
647 for (zcp = list_head(&zfs_cases); zcp != NULL;
648 zcp = list_next(&zfs_cases, zcp)) {
649 if (zcp->zc_data.zc_pool_guid == pool_guid) {
650 pool_found = B_TRUE;
651 pool_load = zcp->zc_when;
652 }
653 if (zcp->zc_data.zc_vdev_guid == vdev_guid)
654 break;
655 }
656
657 /*
658 * Avoid falsely accusing a pool of being faulty. Do so by
659 * not replaying ereports that were generated prior to the
660 * current import. If the failure that generated them was
661 * transient because the device was actually removed but we
662 * didn't receive the normal asynchronous notification, we
663 * don't want to mark it as faulted and potentially panic. If
664 * there is still a problem we'd expect not to be able to
665 * import the pool, or that new ereports will be generated
666 * once the pool is used.
667 */
668 if (pool_found && timeval_earlier(&er_when, &pool_load)) {
669 fmd_hdl_debug(hdl, "ignoring pool %llx, "
670 "ereport time %lld.%lld, pool load time = %lld.%lld",
671 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
672 pool_load.ertv_sec, pool_load.ertv_nsec);
673 zfs_stats.old_drops.fmds_value.ui64++;
674 return;
675 }
676
677 if (!pool_found) {
678 /*
679 * Haven't yet seen this pool, but same situation
680 * may apply.
681 */
682 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
683 struct load_time_arg la;
684
685 la.lt_guid = pool_guid;
686 la.lt_time = &pool_load;
687 la.lt_found = B_FALSE;
688
689 if (zhdl != NULL &&
690 zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
691 la.lt_found == B_TRUE) {
692 pool_found = B_TRUE;
693
694 if (timeval_earlier(&er_when, &pool_load)) {
695 fmd_hdl_debug(hdl, "ignoring pool %llx, "
696 "ereport time %lld.%lld, "
697 "pool load time = %lld.%lld",
698 pool_guid, er_when.ertv_sec,
699 er_when.ertv_nsec, pool_load.ertv_sec,
700 pool_load.ertv_nsec);
701 zfs_stats.old_drops.fmds_value.ui64++;
702 return;
703 }
704 }
705 }
706
707 if (zcp == NULL) {
708 fmd_case_t *cs;
709 zfs_case_data_t data = { 0 };
710
711 /*
712 * If this is one of our 'fake' resource ereports, and there is
713 * no case open, simply discard it.
714 */
715 if (isresource) {
716 zfs_stats.resource_drops.fmds_value.ui64++;
717 fmd_hdl_debug(hdl, "discarding '%s' for vdev %llu",
718 class, vdev_guid);
719 return;
720 }
721
722 /*
723 * Skip tracking some ereports
724 */
725 if (strcmp(class,
726 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 ||
727 strcmp(class,
728 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0) {
729 zfs_stats.resource_drops.fmds_value.ui64++;
730 return;
731 }
732
733 /*
734 * Open a new case.
735 */
736 cs = fmd_case_open(hdl, NULL);
737
738 fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'",
739 vdev_guid, class);
740
741 /*
742 * Initialize the case buffer. To commonize code, we actually
743 * create the buffer with existing data, and then call
744 * zfs_case_unserialize() to instantiate the in-core structure.
745 */
746 fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t));
747
748 data.zc_version = CASE_DATA_VERSION_SERD;
749 data.zc_ena = ena;
750 data.zc_pool_guid = pool_guid;
751 data.zc_vdev_guid = vdev_guid;
752 data.zc_parent_guid = parent_guid;
753 data.zc_pool_state = (int)pool_state;
754
755 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
756
757 zcp = zfs_case_unserialize(hdl, cs);
758 assert(zcp != NULL);
759 if (pool_found)
760 zcp->zc_when = pool_load;
761 }
762
763 if (isresource) {
764 fmd_hdl_debug(hdl, "resource event '%s'", class);
765
766 if (fmd_nvl_class_match(hdl, nvl,
767 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
768 /*
769 * The 'resource.fs.zfs.autoreplace' event indicates
770 * that the pool was loaded with the 'autoreplace'
771 * property set. In this case, any pending device
772 * failures should be ignored, as the asynchronous
773 * autoreplace handling will take care of them.
774 */
775 fmd_case_close(hdl, zcp->zc_case);
776 } else if (fmd_nvl_class_match(hdl, nvl,
777 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
778 /*
779 * The 'resource.fs.zfs.removed' event indicates that
780 * device removal was detected, and the device was
781 * closed asynchronously. If this is the case, we
782 * assume that any recent I/O errors were due to the
783 * device removal, not any fault of the device itself.
784 * We reset the SERD engine, and cancel any pending
785 * timers.
786 */
787 if (zcp->zc_data.zc_has_remove_timer) {
788 fmd_timer_remove(hdl, zcp->zc_remove_timer);
789 zcp->zc_data.zc_has_remove_timer = 0;
790 zfs_case_serialize(zcp);
791 }
792 if (zcp->zc_data.zc_serd_io[0] != '\0')
793 fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io);
794 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
795 fmd_serd_reset(hdl,
796 zcp->zc_data.zc_serd_checksum);
797 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
798 fmd_serd_reset(hdl,
799 zcp->zc_data.zc_serd_slow_io);
800 } else if (fmd_nvl_class_match(hdl, nvl,
801 ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) {
802 uint64_t state = 0;
803
804 if (zcp != NULL &&
805 nvlist_lookup_uint64(nvl,
806 FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 &&
807 state == VDEV_STATE_HEALTHY) {
808 fmd_hdl_debug(hdl, "closing case after a "
809 "device statechange to healthy");
810 fmd_case_close(hdl, zcp->zc_case);
811 }
812 }
813 zfs_stats.resource_drops.fmds_value.ui64++;
814 return;
815 }
816
817 /*
818 * Associate the ereport with this case.
819 */
820 fmd_case_add_ereport(hdl, zcp->zc_case, ep);
821
822 /*
823 * Don't do anything else if this case is already solved.
824 */
825 if (fmd_case_solved(hdl, zcp->zc_case))
826 return;
827
828 if (vdev_guid)
829 fmd_hdl_debug(hdl, "error event '%s', vdev %llu", class,
830 vdev_guid);
831 else
832 fmd_hdl_debug(hdl, "error event '%s'", class);
833
834 /*
835 * Determine if we should solve the case and generate a fault. We solve
836 * a case if:
837 *
838 * a. A pool failed to open (ereport.fs.zfs.pool)
839 * b. A device failed to open (ereport.fs.zfs.pool) while a pool
840 * was up and running.
841 *
842 * We may see a series of ereports associated with a pool open, all
843 * chained together by the same ENA. If the pool open succeeds, then
844 * we'll see no further ereports. To detect when a pool open has
845 * succeeded, we associate a timer with the event. When it expires, we
846 * close the case.
847 */
848 if (fmd_nvl_class_match(hdl, nvl,
849 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
850 /*
851 * Pool level fault. Before solving the case, go through and
852 * close any open device cases that may be pending.
853 */
854 for (dcp = list_head(&zfs_cases); dcp != NULL;
855 dcp = list_next(&zfs_cases, dcp)) {
856 if (dcp->zc_data.zc_pool_guid ==
857 zcp->zc_data.zc_pool_guid &&
858 dcp->zc_data.zc_vdev_guid != 0)
859 fmd_case_close(hdl, dcp->zc_case);
860 }
861
862 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool");
863 } else if (fmd_nvl_class_match(hdl, nvl,
864 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
865 /*
866 * Pool level fault for reading the intent logs.
867 */
868 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay");
869 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
870 /*
871 * Device fault.
872 */
873 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device");
874 } else if (fmd_nvl_class_match(hdl, nvl,
875 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
876 fmd_nvl_class_match(hdl, nvl,
877 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
878 fmd_nvl_class_match(hdl, nvl,
879 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
880 fmd_nvl_class_match(hdl, nvl,
881 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) ||
882 fmd_nvl_class_match(hdl, nvl,
883 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
884 const char *failmode = NULL;
885 boolean_t checkremove = B_FALSE;
886 uint32_t pri = 0;
887
888 /*
889 * If this is a checksum or I/O error, then toss it into the
890 * appropriate SERD engine and check to see if it has fired.
891 * Ideally, we want to do something more sophisticated,
892 * (persistent errors for a single data block, etc). For now,
893 * a single SERD engine is sufficient.
894 */
895 if (fmd_nvl_class_match(hdl, nvl,
896 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
897 if (zcp->zc_data.zc_serd_io[0] == '\0') {
898 if (nvlist_lookup_uint64(nvl,
899 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
900 &io_n) != 0) {
901 io_n = DEFAULT_IO_N;
902 }
903 if (nvlist_lookup_uint64(nvl,
904 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
905 &io_t) != 0) {
906 io_t = DEFAULT_IO_T;
907 }
908 zfs_serd_name(zcp->zc_data.zc_serd_io,
909 pool_guid, vdev_guid, "io");
910 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
911 io_n,
912 SEC2NSEC(io_t));
913 zfs_case_serialize(zcp);
914 }
915 if (zfs_fm_serd_record(hdl, zcp->zc_data.zc_serd_io,
916 ep, zcp, "io error")) {
917 checkremove = B_TRUE;
918 }
919 } else if (fmd_nvl_class_match(hdl, nvl,
920 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY))) {
921 uint64_t slow_io_n, slow_io_t;
922
923 /*
924 * Create a slow io SERD engine when the VDEV has the
925 * 'vdev_slow_io_n' and 'vdev_slow_io_n' properties.
926 */
927 if (zcp->zc_data.zc_serd_slow_io[0] == '\0' &&
928 nvlist_lookup_uint64(nvl,
929 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N,
930 &slow_io_n) == 0 &&
931 nvlist_lookup_uint64(nvl,
932 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T,
933 &slow_io_t) == 0) {
934 zfs_serd_name(zcp->zc_data.zc_serd_slow_io,
935 pool_guid, vdev_guid, "slow_io");
936 fmd_serd_create(hdl,
937 zcp->zc_data.zc_serd_slow_io,
938 slow_io_n,
939 SEC2NSEC(slow_io_t));
940 zfs_case_serialize(zcp);
941 }
942 /* Pass event to SERD engine and see if this triggers */
943 if (zcp->zc_data.zc_serd_slow_io[0] != '\0' &&
944 zfs_fm_serd_record(hdl,
945 zcp->zc_data.zc_serd_slow_io, ep, zcp, "slow io")) {
946 zfs_case_solve(hdl, zcp,
947 "fault.fs.zfs.vdev.slow_io");
948 }
949 } else if (fmd_nvl_class_match(hdl, nvl,
950 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
951 uint64_t flags = 0;
952 int32_t flags32 = 0;
953 /*
954 * We ignore ereports for checksum errors generated by
955 * scrub/resilver I/O to avoid potentially further
956 * degrading the pool while it's being repaired.
957 *
958 * Note that FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS used to
959 * be int32. To allow newer zed to work on older
960 * kernels, if we don't find the flags, we look for
961 * the older ones too.
962 */
963 if (((nvlist_lookup_uint32(nvl,
964 FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY, &pri) == 0) &&
965 (pri == ZIO_PRIORITY_SCRUB ||
966 pri == ZIO_PRIORITY_REBUILD)) ||
967 ((nvlist_lookup_uint64(nvl,
968 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags) == 0) &&
969 (flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) ||
970 ((nvlist_lookup_int32(nvl,
971 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags32) == 0) &&
972 (flags32 & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)))) {
973 fmd_hdl_debug(hdl, "ignoring '%s' for "
974 "scrub/resilver I/O", class);
975 return;
976 }
977
978 if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
979 if (nvlist_lookup_uint64(nvl,
980 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
981 &checksum_n) != 0) {
982 checksum_n = DEFAULT_CHECKSUM_N;
983 }
984 if (nvlist_lookup_uint64(nvl,
985 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
986 &checksum_t) != 0) {
987 checksum_t = DEFAULT_CHECKSUM_T;
988 }
989
990 zfs_serd_name(zcp->zc_data.zc_serd_checksum,
991 pool_guid, vdev_guid, "checksum");
992 fmd_serd_create(hdl,
993 zcp->zc_data.zc_serd_checksum,
994 checksum_n,
995 SEC2NSEC(checksum_t));
996 zfs_case_serialize(zcp);
997 }
998 if (zfs_fm_serd_record(hdl,
999 zcp->zc_data.zc_serd_checksum, ep, zcp,
1000 "checksum")) {
1001 zfs_case_solve(hdl, zcp,
1002 "fault.fs.zfs.vdev.checksum");
1003 }
1004 } else if (fmd_nvl_class_match(hdl, nvl,
1005 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
1006 (nvlist_lookup_string(nvl,
1007 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
1008 failmode != NULL) {
1009 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
1010 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
1011 zfs_case_solve(hdl, zcp,
1012 "fault.fs.zfs.io_failure_continue");
1013 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
1014 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
1015 zfs_case_solve(hdl, zcp,
1016 "fault.fs.zfs.io_failure_wait");
1017 }
1018 } else if (fmd_nvl_class_match(hdl, nvl,
1019 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
1020 #ifndef __linux__
1021 /* This causes an unexpected fault diagnosis on linux */
1022 checkremove = B_TRUE;
1023 #endif
1024 }
1025
1026 /*
1027 * Because I/O errors may be due to device removal, we postpone
1028 * any diagnosis until we're sure that we aren't about to
1029 * receive a 'resource.fs.zfs.removed' event.
1030 */
1031 if (checkremove) {
1032 if (zcp->zc_data.zc_has_remove_timer)
1033 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1034 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
1035 zfs_remove_timeout);
1036 if (!zcp->zc_data.zc_has_remove_timer) {
1037 zcp->zc_data.zc_has_remove_timer = 1;
1038 zfs_case_serialize(zcp);
1039 }
1040 }
1041 }
1042 }
1043
1044 /*
1045 * The timeout is fired when we diagnosed an I/O error, and it was not due to
1046 * device removal (which would cause the timeout to be cancelled).
1047 */
1048 static void
zfs_fm_timeout(fmd_hdl_t * hdl,id_t id,void * data)1049 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
1050 {
1051 zfs_case_t *zcp = data;
1052
1053 if (id == zcp->zc_remove_timer)
1054 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io");
1055 }
1056
1057 /*
1058 * The specified case has been closed and any case-specific
1059 * data structures should be deallocated.
1060 */
1061 static void
zfs_fm_close(fmd_hdl_t * hdl,fmd_case_t * cs)1062 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
1063 {
1064 zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
1065
1066 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
1067 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
1068 if (zcp->zc_data.zc_serd_io[0] != '\0')
1069 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
1070 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
1071 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_slow_io);
1072 if (zcp->zc_data.zc_has_remove_timer)
1073 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1074
1075 list_remove(&zfs_cases, zcp);
1076 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1077 }
1078
1079 static const fmd_hdl_ops_t fmd_ops = {
1080 zfs_fm_recv, /* fmdo_recv */
1081 zfs_fm_timeout, /* fmdo_timeout */
1082 zfs_fm_close, /* fmdo_close */
1083 NULL, /* fmdo_stats */
1084 NULL, /* fmdo_gc */
1085 };
1086
1087 static const fmd_prop_t fmd_props[] = {
1088 { NULL, 0, NULL }
1089 };
1090
1091 static const fmd_hdl_info_t fmd_info = {
1092 "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
1093 };
1094
1095 void
_zfs_diagnosis_init(fmd_hdl_t * hdl)1096 _zfs_diagnosis_init(fmd_hdl_t *hdl)
1097 {
1098 libzfs_handle_t *zhdl;
1099
1100 if ((zhdl = libzfs_init()) == NULL)
1101 return;
1102
1103 list_create(&zfs_cases,
1104 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node));
1105
1106 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
1107 list_destroy(&zfs_cases);
1108 libzfs_fini(zhdl);
1109 return;
1110 }
1111
1112 fmd_hdl_setspecific(hdl, zhdl);
1113
1114 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
1115 sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
1116 }
1117
1118 void
_zfs_diagnosis_fini(fmd_hdl_t * hdl)1119 _zfs_diagnosis_fini(fmd_hdl_t *hdl)
1120 {
1121 zfs_case_t *zcp;
1122 libzfs_handle_t *zhdl;
1123
1124 /*
1125 * Remove all active cases.
1126 */
1127 while ((zcp = list_remove_head(&zfs_cases)) != NULL) {
1128 fmd_hdl_debug(hdl, "removing case ena %llu",
1129 (long long unsigned)zcp->zc_data.zc_ena);
1130 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1131 }
1132
1133 list_destroy(&zfs_cases);
1134
1135 zhdl = fmd_hdl_getspecific(hdl);
1136 libzfs_fini(zhdl);
1137 }
1138