1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
26 * Copyright (c) 2016, Intel Corporation.
27 * Copyright (c) 2023, Klara Inc.
28 */
29
30 #include <stddef.h>
31 #include <string.h>
32 #include <libzfs.h>
33 #include <sys/types.h>
34 #include <sys/time.h>
35 #include <sys/fs/zfs.h>
36 #include <sys/fm/protocol.h>
37 #include <sys/fm/fs/zfs.h>
38 #include <sys/zio.h>
39
40 #include "zfs_agents.h"
41 #include "fmd_api.h"
42
43 /*
44 * Default values for the serd engine when processing checksum or io errors. The
45 * semantics are N <events> in T <seconds>.
46 */
47 #define DEFAULT_CHECKSUM_N 10 /* events */
48 #define DEFAULT_CHECKSUM_T 600 /* seconds */
49 #define DEFAULT_IO_N 10 /* events */
50 #define DEFAULT_IO_T 600 /* seconds */
51
52 #define CASE_GC_TIMEOUT_SECS 43200 /* 12 hours */
53
54 /*
55 * Our serd engines are named in the following format:
56 * 'zfs_<pool_guid>_<vdev_guid>_{checksum,io,slow_io}'
57 * This #define reserves enough space for two 64-bit hex values plus the
58 * length of the longest string.
59 */
60 #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))
61
62 /*
63 * On-disk case structure. This must maintain backwards compatibility with
64 * previous versions of the DE. By default, any members appended to the end
65 * will be filled with zeros if they don't exist in a previous version.
66 */
67 typedef struct zfs_case_data {
68 uint64_t zc_version;
69 uint64_t zc_ena;
70 uint64_t zc_pool_guid;
71 uint64_t zc_vdev_guid;
72 uint64_t zc_parent_guid;
73 int zc_pool_state;
74 char zc_serd_checksum[MAX_SERDLEN];
75 char zc_serd_io[MAX_SERDLEN];
76 char zc_serd_slow_io[MAX_SERDLEN];
77 int zc_has_remove_timer;
78 } zfs_case_data_t;
79
80 /*
81 * Time-of-day
82 */
83 typedef struct er_timeval {
84 uint64_t ertv_sec;
85 uint64_t ertv_nsec;
86 } er_timeval_t;
87
88 /*
89 * In-core case structure.
90 */
91 typedef struct zfs_case {
92 boolean_t zc_present;
93 uint32_t zc_version;
94 zfs_case_data_t zc_data;
95 fmd_case_t *zc_case;
96 list_node_t zc_node;
97 id_t zc_remove_timer;
98 char *zc_fru;
99 er_timeval_t zc_when;
100 } zfs_case_t;
101
102 #define CASE_DATA "data"
103 #define CASE_FRU "fru"
104 #define CASE_DATA_VERSION_INITIAL 1
105 #define CASE_DATA_VERSION_SERD 2
106
107 typedef struct zfs_de_stats {
108 fmd_stat_t old_drops;
109 fmd_stat_t dev_drops;
110 fmd_stat_t vdev_drops;
111 fmd_stat_t import_drops;
112 fmd_stat_t resource_drops;
113 } zfs_de_stats_t;
114
115 zfs_de_stats_t zfs_stats = {
116 { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
117 { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
118 { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
119 { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
120 { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
121 };
122
123 /* wait 15 seconds after a removal */
124 static hrtime_t zfs_remove_timeout = SEC2NSEC(15);
125
126 static list_t zfs_cases;
127
128 #define ZFS_MAKE_RSRC(type) \
129 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
130 #define ZFS_MAKE_EREPORT(type) \
131 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
132
133 static void zfs_purge_cases(fmd_hdl_t *hdl);
134
135 /*
136 * Write out the persistent representation of an active case.
137 */
138 static void
zfs_case_serialize(zfs_case_t * zcp)139 zfs_case_serialize(zfs_case_t *zcp)
140 {
141 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
142 }
143
144 /*
145 * Read back the persistent representation of an active case.
146 */
147 static zfs_case_t *
zfs_case_unserialize(fmd_hdl_t * hdl,fmd_case_t * cp)148 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
149 {
150 zfs_case_t *zcp;
151
152 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
153 zcp->zc_case = cp;
154
155 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
156 sizeof (zcp->zc_data));
157
158 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
159 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
160 return (NULL);
161 }
162
163 /*
164 * fmd_buf_read() will have already zeroed out the remainder of the
165 * buffer, so we don't have to do anything special if the version
166 * doesn't include the SERD engine name.
167 */
168
169 if (zcp->zc_data.zc_has_remove_timer)
170 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
171 NULL, zfs_remove_timeout);
172
173 list_link_init(&zcp->zc_node);
174 list_insert_head(&zfs_cases, zcp);
175
176 fmd_case_setspecific(hdl, cp, zcp);
177
178 return (zcp);
179 }
180
181 /*
182 * Return count of other unique SERD cases under same vdev parent
183 */
184 static uint_t
zfs_other_serd_cases(fmd_hdl_t * hdl,const zfs_case_data_t * zfs_case)185 zfs_other_serd_cases(fmd_hdl_t *hdl, const zfs_case_data_t *zfs_case)
186 {
187 zfs_case_t *zcp;
188 uint_t cases = 0;
189 static hrtime_t next_check = 0;
190
191 /*
192 * Note that plumbing in some external GC would require adding locking,
193 * since most of this module code is not thread safe and assumes there
194 * is only one thread running against the module. So we perform GC here
195 * inline periodically so that future delay induced faults will be
196 * possible once the issue causing multiple vdev delays is resolved.
197 */
198 if (gethrestime_sec() > next_check) {
199 /* Periodically purge old SERD entries and stale cases */
200 fmd_serd_gc(hdl);
201 zfs_purge_cases(hdl);
202 next_check = gethrestime_sec() + CASE_GC_TIMEOUT_SECS;
203 }
204
205 for (zcp = list_head(&zfs_cases); zcp != NULL;
206 zcp = list_next(&zfs_cases, zcp)) {
207 zfs_case_data_t *zcd = &zcp->zc_data;
208
209 /*
210 * must be same pool and parent vdev but different leaf vdev
211 */
212 if (zcd->zc_pool_guid != zfs_case->zc_pool_guid ||
213 zcd->zc_parent_guid != zfs_case->zc_parent_guid ||
214 zcd->zc_vdev_guid == zfs_case->zc_vdev_guid) {
215 continue;
216 }
217
218 /*
219 * Check if there is another active serd case besides zfs_case
220 *
221 * Only one serd engine will be assigned to the case
222 */
223 if (zcd->zc_serd_checksum[0] == zfs_case->zc_serd_checksum[0] &&
224 fmd_serd_active(hdl, zcd->zc_serd_checksum)) {
225 cases++;
226 }
227 if (zcd->zc_serd_io[0] == zfs_case->zc_serd_io[0] &&
228 fmd_serd_active(hdl, zcd->zc_serd_io)) {
229 cases++;
230 }
231 if (zcd->zc_serd_slow_io[0] == zfs_case->zc_serd_slow_io[0] &&
232 fmd_serd_active(hdl, zcd->zc_serd_slow_io)) {
233 cases++;
234 }
235 }
236 return (cases);
237 }
238
239 /*
240 * Iterate over any active cases. If any cases are associated with a pool or
241 * vdev which is no longer present on the system, close the associated case.
242 */
243 static void
zfs_mark_vdev(uint64_t pool_guid,nvlist_t * vd,er_timeval_t * loaded)244 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
245 {
246 uint64_t vdev_guid = 0;
247 uint_t c, children;
248 nvlist_t **child;
249 zfs_case_t *zcp;
250
251 (void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
252
253 /*
254 * Mark any cases associated with this (pool, vdev) pair.
255 */
256 for (zcp = list_head(&zfs_cases); zcp != NULL;
257 zcp = list_next(&zfs_cases, zcp)) {
258 if (zcp->zc_data.zc_pool_guid == pool_guid &&
259 zcp->zc_data.zc_vdev_guid == vdev_guid) {
260 zcp->zc_present = B_TRUE;
261 zcp->zc_when = *loaded;
262 }
263 }
264
265 /*
266 * Iterate over all children.
267 */
268 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
269 &children) == 0) {
270 for (c = 0; c < children; c++)
271 zfs_mark_vdev(pool_guid, child[c], loaded);
272 }
273
274 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
275 &children) == 0) {
276 for (c = 0; c < children; c++)
277 zfs_mark_vdev(pool_guid, child[c], loaded);
278 }
279
280 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
281 &children) == 0) {
282 for (c = 0; c < children; c++)
283 zfs_mark_vdev(pool_guid, child[c], loaded);
284 }
285 }
286
287 static int
zfs_mark_pool(zpool_handle_t * zhp,void * unused)288 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
289 {
290 (void) unused;
291 zfs_case_t *zcp;
292 uint64_t pool_guid;
293 uint64_t *tod;
294 er_timeval_t loaded = { 0 };
295 nvlist_t *config, *vd;
296 uint_t nelem = 0;
297 int ret;
298
299 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
300 /*
301 * Mark any cases associated with just this pool.
302 */
303 for (zcp = list_head(&zfs_cases); zcp != NULL;
304 zcp = list_next(&zfs_cases, zcp)) {
305 if (zcp->zc_data.zc_pool_guid == pool_guid &&
306 zcp->zc_data.zc_vdev_guid == 0)
307 zcp->zc_present = B_TRUE;
308 }
309
310 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
311 zpool_close(zhp);
312 return (-1);
313 }
314
315 (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
316 &tod, &nelem);
317 if (nelem == 2) {
318 loaded.ertv_sec = tod[0];
319 loaded.ertv_nsec = tod[1];
320 for (zcp = list_head(&zfs_cases); zcp != NULL;
321 zcp = list_next(&zfs_cases, zcp)) {
322 if (zcp->zc_data.zc_pool_guid == pool_guid &&
323 zcp->zc_data.zc_vdev_guid == 0) {
324 zcp->zc_when = loaded;
325 }
326 }
327 }
328
329 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
330 if (ret) {
331 zpool_close(zhp);
332 return (-1);
333 }
334
335 zfs_mark_vdev(pool_guid, vd, &loaded);
336
337 zpool_close(zhp);
338
339 return (0);
340 }
341
342 struct load_time_arg {
343 uint64_t lt_guid;
344 er_timeval_t *lt_time;
345 boolean_t lt_found;
346 };
347
348 static int
zpool_find_load_time(zpool_handle_t * zhp,void * arg)349 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
350 {
351 struct load_time_arg *lta = arg;
352 uint64_t pool_guid;
353 uint64_t *tod;
354 nvlist_t *config;
355 uint_t nelem;
356
357 if (lta->lt_found) {
358 zpool_close(zhp);
359 return (0);
360 }
361
362 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
363 if (pool_guid != lta->lt_guid) {
364 zpool_close(zhp);
365 return (0);
366 }
367
368 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
369 zpool_close(zhp);
370 return (-1);
371 }
372
373 if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
374 &tod, &nelem) == 0 && nelem == 2) {
375 lta->lt_found = B_TRUE;
376 lta->lt_time->ertv_sec = tod[0];
377 lta->lt_time->ertv_nsec = tod[1];
378 }
379
380 zpool_close(zhp);
381
382 return (0);
383 }
384
385 static void
zfs_purge_cases(fmd_hdl_t * hdl)386 zfs_purge_cases(fmd_hdl_t *hdl)
387 {
388 zfs_case_t *zcp, *next;
389 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
390
391 /*
392 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No
393 * matter what we do, we're going to have to stomach an O(vdevs * cases)
394 * algorithm. In reality, both quantities are likely so small that
395 * neither will matter. Given that iterating over pools is more
396 * expensive than iterating over the in-memory case list, we opt for a
397 * 'present' flag in each case that starts off cleared. We then iterate
398 * over all pools, marking those that are still present, and removing
399 * those that aren't found.
400 *
401 * Note that we could also construct an FMRI and rely on
402 * fmd_nvl_fmri_present(), but this would end up doing the same search.
403 */
404
405 /*
406 * Mark the cases as not present.
407 */
408 for (zcp = list_head(&zfs_cases); zcp != NULL;
409 zcp = list_next(&zfs_cases, zcp))
410 zcp->zc_present = B_FALSE;
411
412 /*
413 * Iterate over all pools and mark the pools and vdevs found. If this
414 * fails (most probably because we're out of memory), then don't close
415 * any of the cases and we cannot be sure they are accurate.
416 */
417 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
418 return;
419
420 /*
421 * Remove those cases which were not found.
422 */
423 for (zcp = list_head(&zfs_cases); zcp != NULL; zcp = next) {
424 next = list_next(&zfs_cases, zcp);
425 if (!zcp->zc_present)
426 fmd_case_close(hdl, zcp->zc_case);
427 }
428 }
429
430 /*
431 * Construct the name of a serd engine given the pool/vdev GUID and type (io or
432 * checksum).
433 */
434 static void
zfs_serd_name(char * buf,uint64_t pool_guid,uint64_t vdev_guid,const char * type)435 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
436 const char *type)
437 {
438 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s",
439 (long long unsigned int)pool_guid,
440 (long long unsigned int)vdev_guid, type);
441 }
442
443 static void
zfs_case_retire(fmd_hdl_t * hdl,zfs_case_t * zcp)444 zfs_case_retire(fmd_hdl_t *hdl, zfs_case_t *zcp)
445 {
446 fmd_hdl_debug(hdl, "retiring case");
447
448 fmd_case_close(hdl, zcp->zc_case);
449 }
450
451 /*
452 * Solve a given ZFS case. This first checks to make sure the diagnosis is
453 * still valid, as well as cleaning up any pending timer associated with the
454 * case.
455 */
456 static void
zfs_case_solve(fmd_hdl_t * hdl,zfs_case_t * zcp,const char * faultname)457 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname)
458 {
459 nvlist_t *detector, *fault;
460 boolean_t serialize;
461 nvlist_t *fru = NULL;
462 fmd_hdl_debug(hdl, "solving fault '%s'", faultname);
463
464 /*
465 * Construct the detector from the case data. The detector is in the
466 * ZFS scheme, and is either the pool or the vdev, depending on whether
467 * this is a vdev or pool fault.
468 */
469 detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
470
471 (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
472 (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
473 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
474 zcp->zc_data.zc_pool_guid);
475 if (zcp->zc_data.zc_vdev_guid != 0) {
476 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
477 zcp->zc_data.zc_vdev_guid);
478 }
479
480 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
481 fru, detector);
482 fmd_case_add_suspect(hdl, zcp->zc_case, fault);
483
484 nvlist_free(fru);
485
486 fmd_case_solve(hdl, zcp->zc_case);
487
488 serialize = B_FALSE;
489 if (zcp->zc_data.zc_has_remove_timer) {
490 fmd_timer_remove(hdl, zcp->zc_remove_timer);
491 zcp->zc_data.zc_has_remove_timer = 0;
492 serialize = B_TRUE;
493 }
494 if (serialize)
495 zfs_case_serialize(zcp);
496
497 nvlist_free(detector);
498 }
499
500 static boolean_t
timeval_earlier(er_timeval_t * a,er_timeval_t * b)501 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
502 {
503 return (a->ertv_sec < b->ertv_sec ||
504 (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
505 }
506
507 static void
zfs_ereport_when(fmd_hdl_t * hdl,nvlist_t * nvl,er_timeval_t * when)508 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
509 {
510 (void) hdl;
511 int64_t *tod;
512 uint_t nelem;
513
514 if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod,
515 &nelem) == 0 && nelem == 2) {
516 when->ertv_sec = tod[0];
517 when->ertv_nsec = tod[1];
518 } else {
519 when->ertv_sec = when->ertv_nsec = UINT64_MAX;
520 }
521 }
522
523 /*
524 * Record the specified event in the SERD engine and return a
525 * boolean value indicating whether or not the engine fired as
526 * the result of inserting this event.
527 *
528 * When the pool has similar active cases on other vdevs, then
529 * the fired state is disregarded and the case is retired.
530 */
531 static int
zfs_fm_serd_record(fmd_hdl_t * hdl,const char * name,fmd_event_t * ep,zfs_case_t * zcp,const char * err_type)532 zfs_fm_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep,
533 zfs_case_t *zcp, const char *err_type)
534 {
535 int fired = fmd_serd_record(hdl, name, ep);
536 int peers = 0;
537
538 if (fired && (peers = zfs_other_serd_cases(hdl, &zcp->zc_data)) > 0) {
539 fmd_hdl_debug(hdl, "pool %llu is tracking %d other %s cases "
540 "-- skip faulting the vdev %llu",
541 (u_longlong_t)zcp->zc_data.zc_pool_guid,
542 peers, err_type,
543 (u_longlong_t)zcp->zc_data.zc_vdev_guid);
544 zfs_case_retire(hdl, zcp);
545 fired = 0;
546 }
547
548 return (fired);
549 }
550
551 /*
552 * Main fmd entry point.
553 */
554 static void
zfs_fm_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)555 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
556 {
557 zfs_case_t *zcp, *dcp;
558 int32_t pool_state;
559 uint64_t ena, pool_guid, vdev_guid, parent_guid;
560 uint64_t checksum_n, checksum_t;
561 uint64_t io_n, io_t;
562 er_timeval_t pool_load;
563 er_timeval_t er_when;
564 nvlist_t *detector;
565 boolean_t pool_found = B_FALSE;
566 boolean_t isresource;
567 const char *type;
568
569 /*
570 * We subscribe to notifications for vdev or pool removal. In these
571 * cases, there may be cases that no longer apply. Purge any cases
572 * that no longer apply.
573 */
574 if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) {
575 fmd_hdl_debug(hdl, "purging orphaned cases from %s",
576 strrchr(class, '.') + 1);
577 zfs_purge_cases(hdl);
578 zfs_stats.resource_drops.fmds_value.ui64++;
579 return;
580 }
581
582 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
583
584 if (isresource) {
585 /*
586 * For resources, we don't have a normal payload.
587 */
588 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
589 &vdev_guid) != 0)
590 pool_state = SPA_LOAD_OPEN;
591 else
592 pool_state = SPA_LOAD_NONE;
593 detector = NULL;
594 } else {
595 (void) nvlist_lookup_nvlist(nvl,
596 FM_EREPORT_DETECTOR, &detector);
597 (void) nvlist_lookup_int32(nvl,
598 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
599 }
600
601 /*
602 * We also ignore all ereports generated during an import of a pool,
603 * since the only possible fault (.pool) would result in import failure,
604 * and hence no persistent fault. Some day we may want to do something
605 * with these ereports, so we continue generating them internally.
606 */
607 if (pool_state == SPA_LOAD_IMPORT) {
608 zfs_stats.import_drops.fmds_value.ui64++;
609 fmd_hdl_debug(hdl, "ignoring '%s' during import", class);
610 return;
611 }
612
613 /*
614 * Device I/O errors are ignored during pool open.
615 */
616 if (pool_state == SPA_LOAD_OPEN &&
617 (fmd_nvl_class_match(hdl, nvl,
618 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
619 fmd_nvl_class_match(hdl, nvl,
620 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
621 fmd_nvl_class_match(hdl, nvl,
622 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
623 fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class);
624 zfs_stats.dev_drops.fmds_value.ui64++;
625 return;
626 }
627
628 /*
629 * We ignore ereports for anything except disks and files.
630 */
631 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
632 &type) == 0) {
633 if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
634 strcmp(type, VDEV_TYPE_FILE) != 0) {
635 zfs_stats.vdev_drops.fmds_value.ui64++;
636 return;
637 }
638 }
639
640 /*
641 * Determine if this ereport corresponds to an open case.
642 * Each vdev or pool can have a single case.
643 */
644 (void) nvlist_lookup_uint64(nvl,
645 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
646 if (nvlist_lookup_uint64(nvl,
647 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
648 vdev_guid = 0;
649 if (nvlist_lookup_uint64(nvl,
650 FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID, &parent_guid) != 0)
651 parent_guid = 0;
652 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
653 ena = 0;
654
655 zfs_ereport_when(hdl, nvl, &er_when);
656
657 for (zcp = list_head(&zfs_cases); zcp != NULL;
658 zcp = list_next(&zfs_cases, zcp)) {
659 if (zcp->zc_data.zc_pool_guid == pool_guid) {
660 pool_found = B_TRUE;
661 pool_load = zcp->zc_when;
662 }
663 if (zcp->zc_data.zc_vdev_guid == vdev_guid)
664 break;
665 }
666
667 /*
668 * Avoid falsely accusing a pool of being faulty. Do so by
669 * not replaying ereports that were generated prior to the
670 * current import. If the failure that generated them was
671 * transient because the device was actually removed but we
672 * didn't receive the normal asynchronous notification, we
673 * don't want to mark it as faulted and potentially panic. If
674 * there is still a problem we'd expect not to be able to
675 * import the pool, or that new ereports will be generated
676 * once the pool is used.
677 */
678 if (pool_found && timeval_earlier(&er_when, &pool_load)) {
679 fmd_hdl_debug(hdl, "ignoring pool %llx, "
680 "ereport time %lld.%lld, pool load time = %lld.%lld",
681 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
682 pool_load.ertv_sec, pool_load.ertv_nsec);
683 zfs_stats.old_drops.fmds_value.ui64++;
684 return;
685 }
686
687 if (!pool_found) {
688 /*
689 * Haven't yet seen this pool, but same situation
690 * may apply.
691 */
692 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
693 struct load_time_arg la;
694
695 la.lt_guid = pool_guid;
696 la.lt_time = &pool_load;
697 la.lt_found = B_FALSE;
698
699 if (zhdl != NULL &&
700 zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
701 la.lt_found == B_TRUE) {
702 pool_found = B_TRUE;
703
704 if (timeval_earlier(&er_when, &pool_load)) {
705 fmd_hdl_debug(hdl, "ignoring pool %llx, "
706 "ereport time %lld.%lld, "
707 "pool load time = %lld.%lld",
708 pool_guid, er_when.ertv_sec,
709 er_when.ertv_nsec, pool_load.ertv_sec,
710 pool_load.ertv_nsec);
711 zfs_stats.old_drops.fmds_value.ui64++;
712 return;
713 }
714 }
715 }
716
717 if (zcp == NULL) {
718 fmd_case_t *cs;
719 zfs_case_data_t data = { 0 };
720
721 /*
722 * If this is one of our 'fake' resource ereports, and there is
723 * no case open, simply discard it.
724 */
725 if (isresource) {
726 zfs_stats.resource_drops.fmds_value.ui64++;
727 fmd_hdl_debug(hdl, "discarding '%s' for vdev %llu",
728 class, vdev_guid);
729 return;
730 }
731
732 /*
733 * Skip tracking some ereports
734 */
735 if (strcmp(class,
736 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 ||
737 strcmp(class,
738 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0) {
739 zfs_stats.resource_drops.fmds_value.ui64++;
740 return;
741 }
742
743 /*
744 * Open a new case.
745 */
746 cs = fmd_case_open(hdl, NULL);
747
748 fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'",
749 vdev_guid, class);
750
751 /*
752 * Initialize the case buffer. To commonize code, we actually
753 * create the buffer with existing data, and then call
754 * zfs_case_unserialize() to instantiate the in-core structure.
755 */
756 fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t));
757
758 data.zc_version = CASE_DATA_VERSION_SERD;
759 data.zc_ena = ena;
760 data.zc_pool_guid = pool_guid;
761 data.zc_vdev_guid = vdev_guid;
762 data.zc_parent_guid = parent_guid;
763 data.zc_pool_state = (int)pool_state;
764
765 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
766
767 zcp = zfs_case_unserialize(hdl, cs);
768 assert(zcp != NULL);
769 if (pool_found)
770 zcp->zc_when = pool_load;
771 }
772
773 if (isresource) {
774 fmd_hdl_debug(hdl, "resource event '%s'", class);
775
776 if (fmd_nvl_class_match(hdl, nvl,
777 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
778 /*
779 * The 'resource.fs.zfs.autoreplace' event indicates
780 * that the pool was loaded with the 'autoreplace'
781 * property set. In this case, any pending device
782 * failures should be ignored, as the asynchronous
783 * autoreplace handling will take care of them.
784 */
785 fmd_case_close(hdl, zcp->zc_case);
786 } else if (fmd_nvl_class_match(hdl, nvl,
787 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
788 /*
789 * The 'resource.fs.zfs.removed' event indicates that
790 * device removal was detected, and the device was
791 * closed asynchronously. If this is the case, we
792 * assume that any recent I/O errors were due to the
793 * device removal, not any fault of the device itself.
794 * We reset the SERD engine, and cancel any pending
795 * timers.
796 */
797 if (zcp->zc_data.zc_has_remove_timer) {
798 fmd_timer_remove(hdl, zcp->zc_remove_timer);
799 zcp->zc_data.zc_has_remove_timer = 0;
800 zfs_case_serialize(zcp);
801 }
802 if (zcp->zc_data.zc_serd_io[0] != '\0')
803 fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io);
804 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
805 fmd_serd_reset(hdl,
806 zcp->zc_data.zc_serd_checksum);
807 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
808 fmd_serd_reset(hdl,
809 zcp->zc_data.zc_serd_slow_io);
810 } else if (fmd_nvl_class_match(hdl, nvl,
811 ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) {
812 uint64_t state = 0;
813
814 if (zcp != NULL &&
815 nvlist_lookup_uint64(nvl,
816 FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 &&
817 state == VDEV_STATE_HEALTHY) {
818 fmd_hdl_debug(hdl, "closing case after a "
819 "device statechange to healthy");
820 fmd_case_close(hdl, zcp->zc_case);
821 }
822 }
823 zfs_stats.resource_drops.fmds_value.ui64++;
824 return;
825 }
826
827 /*
828 * Associate the ereport with this case.
829 */
830 fmd_case_add_ereport(hdl, zcp->zc_case, ep);
831
832 /*
833 * Don't do anything else if this case is already solved.
834 */
835 if (fmd_case_solved(hdl, zcp->zc_case))
836 return;
837
838 if (vdev_guid)
839 fmd_hdl_debug(hdl, "error event '%s', vdev %llu", class,
840 vdev_guid);
841 else
842 fmd_hdl_debug(hdl, "error event '%s'", class);
843
844 /*
845 * Determine if we should solve the case and generate a fault. We solve
846 * a case if:
847 *
848 * a. A pool failed to open (ereport.fs.zfs.pool)
849 * b. A device failed to open (ereport.fs.zfs.pool) while a pool
850 * was up and running.
851 *
852 * We may see a series of ereports associated with a pool open, all
853 * chained together by the same ENA. If the pool open succeeds, then
854 * we'll see no further ereports. To detect when a pool open has
855 * succeeded, we associate a timer with the event. When it expires, we
856 * close the case.
857 */
858 if (fmd_nvl_class_match(hdl, nvl,
859 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
860 /*
861 * Pool level fault. Before solving the case, go through and
862 * close any open device cases that may be pending.
863 */
864 for (dcp = list_head(&zfs_cases); dcp != NULL;
865 dcp = list_next(&zfs_cases, dcp)) {
866 if (dcp->zc_data.zc_pool_guid ==
867 zcp->zc_data.zc_pool_guid &&
868 dcp->zc_data.zc_vdev_guid != 0)
869 fmd_case_close(hdl, dcp->zc_case);
870 }
871
872 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool");
873 } else if (fmd_nvl_class_match(hdl, nvl,
874 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
875 /*
876 * Pool level fault for reading the intent logs.
877 */
878 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay");
879 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
880 /*
881 * Device fault.
882 */
883 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device");
884 } else if (fmd_nvl_class_match(hdl, nvl,
885 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
886 fmd_nvl_class_match(hdl, nvl,
887 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
888 fmd_nvl_class_match(hdl, nvl,
889 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
890 fmd_nvl_class_match(hdl, nvl,
891 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) ||
892 fmd_nvl_class_match(hdl, nvl,
893 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
894 const char *failmode = NULL;
895 boolean_t checkremove = B_FALSE;
896 uint32_t pri = 0;
897
898 /*
899 * If this is a checksum or I/O error, then toss it into the
900 * appropriate SERD engine and check to see if it has fired.
901 * Ideally, we want to do something more sophisticated,
902 * (persistent errors for a single data block, etc). For now,
903 * a single SERD engine is sufficient.
904 */
905 if (fmd_nvl_class_match(hdl, nvl,
906 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
907 if (zcp->zc_data.zc_serd_io[0] == '\0') {
908 if (nvlist_lookup_uint64(nvl,
909 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
910 &io_n) != 0) {
911 io_n = DEFAULT_IO_N;
912 }
913 if (nvlist_lookup_uint64(nvl,
914 FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
915 &io_t) != 0) {
916 io_t = DEFAULT_IO_T;
917 }
918 zfs_serd_name(zcp->zc_data.zc_serd_io,
919 pool_guid, vdev_guid, "io");
920 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
921 io_n,
922 SEC2NSEC(io_t));
923 zfs_case_serialize(zcp);
924 }
925 if (zfs_fm_serd_record(hdl, zcp->zc_data.zc_serd_io,
926 ep, zcp, "io error")) {
927 checkremove = B_TRUE;
928 }
929 } else if (fmd_nvl_class_match(hdl, nvl,
930 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY))) {
931 uint64_t slow_io_n, slow_io_t;
932
933 /*
934 * Create a slow io SERD engine when the VDEV has the
935 * 'vdev_slow_io_n' and 'vdev_slow_io_n' properties.
936 */
937 if (zcp->zc_data.zc_serd_slow_io[0] == '\0' &&
938 nvlist_lookup_uint64(nvl,
939 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N,
940 &slow_io_n) == 0 &&
941 nvlist_lookup_uint64(nvl,
942 FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T,
943 &slow_io_t) == 0) {
944 zfs_serd_name(zcp->zc_data.zc_serd_slow_io,
945 pool_guid, vdev_guid, "slow_io");
946 fmd_serd_create(hdl,
947 zcp->zc_data.zc_serd_slow_io,
948 slow_io_n,
949 SEC2NSEC(slow_io_t));
950 zfs_case_serialize(zcp);
951 }
952 /* Pass event to SERD engine and see if this triggers */
953 if (zcp->zc_data.zc_serd_slow_io[0] != '\0' &&
954 zfs_fm_serd_record(hdl,
955 zcp->zc_data.zc_serd_slow_io, ep, zcp, "slow io")) {
956 zfs_case_solve(hdl, zcp,
957 "fault.fs.zfs.vdev.slow_io");
958 }
959 } else if (fmd_nvl_class_match(hdl, nvl,
960 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
961 uint64_t flags = 0;
962 int32_t flags32 = 0;
963 /*
964 * We ignore ereports for checksum errors generated by
965 * scrub/resilver I/O to avoid potentially further
966 * degrading the pool while it's being repaired.
967 *
968 * Note that FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS used to
969 * be int32. To allow newer zed to work on older
970 * kernels, if we don't find the flags, we look for
971 * the older ones too.
972 */
973 if (((nvlist_lookup_uint32(nvl,
974 FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY, &pri) == 0) &&
975 (pri == ZIO_PRIORITY_SCRUB ||
976 pri == ZIO_PRIORITY_REBUILD)) ||
977 ((nvlist_lookup_uint64(nvl,
978 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags) == 0) &&
979 (flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) ||
980 ((nvlist_lookup_int32(nvl,
981 FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags32) == 0) &&
982 (flags32 & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)))) {
983 fmd_hdl_debug(hdl, "ignoring '%s' for "
984 "scrub/resilver I/O", class);
985 return;
986 }
987
988 if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
989 if (nvlist_lookup_uint64(nvl,
990 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
991 &checksum_n) != 0) {
992 checksum_n = DEFAULT_CHECKSUM_N;
993 }
994 if (nvlist_lookup_uint64(nvl,
995 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
996 &checksum_t) != 0) {
997 checksum_t = DEFAULT_CHECKSUM_T;
998 }
999
1000 zfs_serd_name(zcp->zc_data.zc_serd_checksum,
1001 pool_guid, vdev_guid, "checksum");
1002 fmd_serd_create(hdl,
1003 zcp->zc_data.zc_serd_checksum,
1004 checksum_n,
1005 SEC2NSEC(checksum_t));
1006 zfs_case_serialize(zcp);
1007 }
1008 if (zfs_fm_serd_record(hdl,
1009 zcp->zc_data.zc_serd_checksum, ep, zcp,
1010 "checksum")) {
1011 zfs_case_solve(hdl, zcp,
1012 "fault.fs.zfs.vdev.checksum");
1013 }
1014 } else if (fmd_nvl_class_match(hdl, nvl,
1015 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
1016 (nvlist_lookup_string(nvl,
1017 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
1018 failmode != NULL) {
1019 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
1020 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
1021 zfs_case_solve(hdl, zcp,
1022 "fault.fs.zfs.io_failure_continue");
1023 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
1024 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
1025 zfs_case_solve(hdl, zcp,
1026 "fault.fs.zfs.io_failure_wait");
1027 }
1028 } else if (fmd_nvl_class_match(hdl, nvl,
1029 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
1030 #ifndef __linux__
1031 /* This causes an unexpected fault diagnosis on linux */
1032 checkremove = B_TRUE;
1033 #endif
1034 }
1035
1036 /*
1037 * Because I/O errors may be due to device removal, we postpone
1038 * any diagnosis until we're sure that we aren't about to
1039 * receive a 'resource.fs.zfs.removed' event.
1040 */
1041 if (checkremove) {
1042 if (zcp->zc_data.zc_has_remove_timer)
1043 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1044 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
1045 zfs_remove_timeout);
1046 if (!zcp->zc_data.zc_has_remove_timer) {
1047 zcp->zc_data.zc_has_remove_timer = 1;
1048 zfs_case_serialize(zcp);
1049 }
1050 }
1051 }
1052 }
1053
1054 /*
1055 * The timeout is fired when we diagnosed an I/O error, and it was not due to
1056 * device removal (which would cause the timeout to be cancelled).
1057 */
1058 static void
zfs_fm_timeout(fmd_hdl_t * hdl,id_t id,void * data)1059 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
1060 {
1061 zfs_case_t *zcp = data;
1062
1063 if (id == zcp->zc_remove_timer)
1064 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io");
1065 }
1066
1067 /*
1068 * The specified case has been closed and any case-specific
1069 * data structures should be deallocated.
1070 */
1071 static void
zfs_fm_close(fmd_hdl_t * hdl,fmd_case_t * cs)1072 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
1073 {
1074 zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
1075
1076 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
1077 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
1078 if (zcp->zc_data.zc_serd_io[0] != '\0')
1079 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
1080 if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
1081 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_slow_io);
1082 if (zcp->zc_data.zc_has_remove_timer)
1083 fmd_timer_remove(hdl, zcp->zc_remove_timer);
1084
1085 list_remove(&zfs_cases, zcp);
1086 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1087 }
1088
1089 static const fmd_hdl_ops_t fmd_ops = {
1090 zfs_fm_recv, /* fmdo_recv */
1091 zfs_fm_timeout, /* fmdo_timeout */
1092 zfs_fm_close, /* fmdo_close */
1093 NULL, /* fmdo_stats */
1094 NULL, /* fmdo_gc */
1095 };
1096
1097 static const fmd_prop_t fmd_props[] = {
1098 { NULL, 0, NULL }
1099 };
1100
1101 static const fmd_hdl_info_t fmd_info = {
1102 "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
1103 };
1104
1105 void
_zfs_diagnosis_init(fmd_hdl_t * hdl)1106 _zfs_diagnosis_init(fmd_hdl_t *hdl)
1107 {
1108 libzfs_handle_t *zhdl;
1109
1110 if ((zhdl = libzfs_init()) == NULL)
1111 return;
1112
1113 list_create(&zfs_cases,
1114 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node));
1115
1116 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
1117 list_destroy(&zfs_cases);
1118 libzfs_fini(zhdl);
1119 return;
1120 }
1121
1122 fmd_hdl_setspecific(hdl, zhdl);
1123
1124 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
1125 sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
1126 }
1127
1128 void
_zfs_diagnosis_fini(fmd_hdl_t * hdl)1129 _zfs_diagnosis_fini(fmd_hdl_t *hdl)
1130 {
1131 zfs_case_t *zcp;
1132 libzfs_handle_t *zhdl;
1133
1134 /*
1135 * Remove all active cases.
1136 */
1137 while ((zcp = list_remove_head(&zfs_cases)) != NULL) {
1138 fmd_hdl_debug(hdl, "removing case ena %llu",
1139 (long long unsigned)zcp->zc_data.zc_ena);
1140 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1141 }
1142
1143 list_destroy(&zfs_cases);
1144
1145 zhdl = fmd_hdl_getspecific(hdl);
1146 libzfs_fini(zhdl);
1147 }
1148