1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2012 by Delphix. All rights reserved.
26 * Copyright (c) 2013 Steven Hartland. All rights reserved.
27 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
28 */
29
30 /*
31 * This file contains the functions which analyze the status of a pool. This
32 * include both the status of an active pool, as well as the status exported
33 * pools. Returns one of the ZPOOL_STATUS_* defines describing the status of
34 * the pool. This status is independent (to a certain degree) from the state of
35 * the pool. A pool's state describes only whether or not it is capable of
36 * providing the necessary fault tolerance for data. The status describes the
37 * overall status of devices. A pool that is online can still have a device
38 * that is experiencing errors.
39 *
40 * Only a subset of the possible faults can be detected using 'zpool status',
41 * and not all possible errors correspond to a FMA message ID. The explanation
42 * is left up to the caller, depending on whether it is a live pool or an
43 * import.
44 */
45
46 #include <libzfs.h>
47 #include <libzutil.h>
48 #include <stdlib.h>
49 #include <string.h>
50 #include <unistd.h>
51 #include <sys/systeminfo.h>
52 #include "libzfs_impl.h"
53 #include "zfeature_common.h"
54
55 /*
56 * Message ID table. This must be kept in sync with the ZPOOL_STATUS_* defines
57 * in include/libzfs.h. Note that there are some status results which go past
58 * the end of this table, and hence have no associated message ID.
59 */
60 static const char *const zfs_msgid_table[] = {
61 "ZFS-8000-14", /* ZPOOL_STATUS_CORRUPT_CACHE */
62 "ZFS-8000-2Q", /* ZPOOL_STATUS_MISSING_DEV_R */
63 "ZFS-8000-3C", /* ZPOOL_STATUS_MISSING_DEV_NR */
64 "ZFS-8000-4J", /* ZPOOL_STATUS_CORRUPT_LABEL_R */
65 "ZFS-8000-5E", /* ZPOOL_STATUS_CORRUPT_LABEL_NR */
66 "ZFS-8000-6X", /* ZPOOL_STATUS_BAD_GUID_SUM */
67 "ZFS-8000-72", /* ZPOOL_STATUS_CORRUPT_POOL */
68 "ZFS-8000-8A", /* ZPOOL_STATUS_CORRUPT_DATA */
69 "ZFS-8000-9P", /* ZPOOL_STATUS_FAILING_DEV */
70 "ZFS-8000-A5", /* ZPOOL_STATUS_VERSION_NEWER */
71 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_MISMATCH */
72 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_ACTIVE */
73 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_REQUIRED */
74 "ZFS-8000-HC", /* ZPOOL_STATUS_IO_FAILURE_WAIT */
75 "ZFS-8000-JQ", /* ZPOOL_STATUS_IO_FAILURE_CONTINUE */
76 "ZFS-8000-MM", /* ZPOOL_STATUS_IO_FAILURE_MMP */
77 "ZFS-8000-K4", /* ZPOOL_STATUS_BAD_LOG */
78 "ZFS-8000-ER", /* ZPOOL_STATUS_ERRATA */
79 /*
80 * The following results have no message ID.
81 * ZPOOL_STATUS_UNSUP_FEAT_READ
82 * ZPOOL_STATUS_UNSUP_FEAT_WRITE
83 * ZPOOL_STATUS_FAULTED_DEV_R
84 * ZPOOL_STATUS_FAULTED_DEV_NR
85 * ZPOOL_STATUS_VERSION_OLDER
86 * ZPOOL_STATUS_FEAT_DISABLED
87 * ZPOOL_STATUS_RESILVERING
88 * ZPOOL_STATUS_OFFLINE_DEV
89 * ZPOOL_STATUS_REMOVED_DEV
90 * ZPOOL_STATUS_REBUILDING
91 * ZPOOL_STATUS_REBUILD_SCRUB
92 * ZPOOL_STATUS_COMPATIBILITY_ERR
93 * ZPOOL_STATUS_INCOMPATIBLE_FEAT
94 * ZPOOL_STATUS_OK
95 */
96 };
97
98 #define NMSGID (sizeof (zfs_msgid_table) / sizeof (zfs_msgid_table[0]))
99
100 static int
vdev_missing(vdev_stat_t * vs,uint_t vsc)101 vdev_missing(vdev_stat_t *vs, uint_t vsc)
102 {
103 (void) vsc;
104 return (vs->vs_state == VDEV_STATE_CANT_OPEN &&
105 vs->vs_aux == VDEV_AUX_OPEN_FAILED);
106 }
107
108 static int
vdev_faulted(vdev_stat_t * vs,uint_t vsc)109 vdev_faulted(vdev_stat_t *vs, uint_t vsc)
110 {
111 (void) vsc;
112 return (vs->vs_state == VDEV_STATE_FAULTED);
113 }
114
115 static int
vdev_errors(vdev_stat_t * vs,uint_t vsc)116 vdev_errors(vdev_stat_t *vs, uint_t vsc)
117 {
118 (void) vsc;
119 return (vs->vs_state == VDEV_STATE_DEGRADED ||
120 vs->vs_read_errors != 0 || vs->vs_write_errors != 0 ||
121 vs->vs_checksum_errors != 0);
122 }
123
124 static int
vdev_broken(vdev_stat_t * vs,uint_t vsc)125 vdev_broken(vdev_stat_t *vs, uint_t vsc)
126 {
127 (void) vsc;
128 return (vs->vs_state == VDEV_STATE_CANT_OPEN);
129 }
130
131 static int
vdev_offlined(vdev_stat_t * vs,uint_t vsc)132 vdev_offlined(vdev_stat_t *vs, uint_t vsc)
133 {
134 (void) vsc;
135 return (vs->vs_state == VDEV_STATE_OFFLINE);
136 }
137
138 static int
vdev_removed(vdev_stat_t * vs,uint_t vsc)139 vdev_removed(vdev_stat_t *vs, uint_t vsc)
140 {
141 (void) vsc;
142 return (vs->vs_state == VDEV_STATE_REMOVED);
143 }
144
145 static int
vdev_non_native_ashift(vdev_stat_t * vs,uint_t vsc)146 vdev_non_native_ashift(vdev_stat_t *vs, uint_t vsc)
147 {
148 if (getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") != NULL)
149 return (0);
150
151 return (VDEV_STAT_VALID(vs_physical_ashift, vsc) &&
152 vs->vs_configured_ashift < vs->vs_physical_ashift);
153 }
154
155 /*
156 * Detect if any leaf devices that have seen errors or could not be opened.
157 */
158 static boolean_t
find_vdev_problem(nvlist_t * vdev,int (* func)(vdev_stat_t *,uint_t),boolean_t ignore_replacing)159 find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t),
160 boolean_t ignore_replacing)
161 {
162 nvlist_t **child;
163 uint_t c, children;
164
165 /*
166 * Ignore problems within a 'replacing' vdev, since we're presumably in
167 * the process of repairing any such errors, and don't want to call them
168 * out again. We'll pick up the fact that a resilver is happening
169 * later.
170 */
171 if (ignore_replacing == B_TRUE) {
172 const char *type = fnvlist_lookup_string(vdev,
173 ZPOOL_CONFIG_TYPE);
174 if (strcmp(type, VDEV_TYPE_REPLACING) == 0)
175 return (B_FALSE);
176 }
177
178 if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_CHILDREN, &child,
179 &children) == 0) {
180 for (c = 0; c < children; c++)
181 if (find_vdev_problem(child[c], func, ignore_replacing))
182 return (B_TRUE);
183 } else {
184 uint_t vsc;
185 vdev_stat_t *vs = (vdev_stat_t *)fnvlist_lookup_uint64_array(
186 vdev, ZPOOL_CONFIG_VDEV_STATS, &vsc);
187 if (func(vs, vsc) != 0)
188 return (B_TRUE);
189 }
190
191 /*
192 * Check any L2 cache devs
193 */
194 if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_L2CACHE, &child,
195 &children) == 0) {
196 for (c = 0; c < children; c++)
197 if (find_vdev_problem(child[c], func, ignore_replacing))
198 return (B_TRUE);
199 }
200
201 return (B_FALSE);
202 }
203
204 /*
205 * Active pool health status.
206 *
207 * To determine the status for a pool, we make several passes over the config,
208 * picking the most egregious error we find. In order of importance, we do the
209 * following:
210 *
211 * - Check for a complete and valid configuration
212 * - Look for any faulted or missing devices in a non-replicated config
213 * - Check for any data errors
214 * - Check for any faulted or missing devices in a replicated config
215 * - Look for any devices showing errors
216 * - Check for any resilvering or rebuilding devices
217 *
218 * There can obviously be multiple errors within a single pool, so this routine
219 * only picks the most damaging of all the current errors to report.
220 */
221 static zpool_status_t
check_status(nvlist_t * config,boolean_t isimport,zpool_errata_t * erratap,const char * compat)222 check_status(nvlist_t *config, boolean_t isimport,
223 zpool_errata_t *erratap, const char *compat)
224 {
225 pool_scan_stat_t *ps = NULL;
226 uint_t vsc, psc;
227 uint64_t suspended;
228 uint64_t hostid = 0;
229 uint64_t errata = 0;
230 unsigned long system_hostid = get_system_hostid();
231
232 uint64_t version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
233 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
234 ZPOOL_CONFIG_VDEV_TREE);
235 vdev_stat_t *vs = (vdev_stat_t *)fnvlist_lookup_uint64_array(nvroot,
236 ZPOOL_CONFIG_VDEV_STATS, &vsc);
237 uint64_t stateval = fnvlist_lookup_uint64(config,
238 ZPOOL_CONFIG_POOL_STATE);
239
240 /*
241 * Currently resilvering a vdev
242 */
243 (void) nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
244 (uint64_t **)&ps, &psc);
245 if (ps != NULL && ps->pss_func == POOL_SCAN_RESILVER &&
246 ps->pss_state == DSS_SCANNING)
247 return (ZPOOL_STATUS_RESILVERING);
248
249 /*
250 * Currently rebuilding a vdev, check top-level vdevs.
251 */
252 vdev_rebuild_stat_t *vrs = NULL;
253 nvlist_t **child;
254 uint_t c, i, children;
255 uint64_t rebuild_end_time = 0;
256 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
257 &child, &children) == 0) {
258 for (c = 0; c < children; c++) {
259 if ((nvlist_lookup_uint64_array(child[c],
260 ZPOOL_CONFIG_REBUILD_STATS,
261 (uint64_t **)&vrs, &i) == 0) && (vrs != NULL)) {
262 uint64_t state = vrs->vrs_state;
263
264 if (state == VDEV_REBUILD_ACTIVE) {
265 return (ZPOOL_STATUS_REBUILDING);
266 } else if (state == VDEV_REBUILD_COMPLETE &&
267 vrs->vrs_end_time > rebuild_end_time) {
268 rebuild_end_time = vrs->vrs_end_time;
269 }
270 }
271 }
272
273 /*
274 * If we can determine when the last scrub was run, and it
275 * was before the last rebuild completed, then recommend
276 * that the pool be scrubbed to verify all checksums. When
277 * ps is NULL we can infer the pool has never been scrubbed.
278 */
279 if (rebuild_end_time > 0) {
280 if (ps != NULL) {
281 if ((ps->pss_state == DSS_FINISHED &&
282 ps->pss_func == POOL_SCAN_SCRUB &&
283 rebuild_end_time > ps->pss_end_time) ||
284 ps->pss_state == DSS_NONE)
285 return (ZPOOL_STATUS_REBUILD_SCRUB);
286 } else {
287 return (ZPOOL_STATUS_REBUILD_SCRUB);
288 }
289 }
290 }
291
292 /*
293 * The multihost property is set and the pool may be active.
294 */
295 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
296 vs->vs_aux == VDEV_AUX_ACTIVE) {
297 mmp_state_t mmp_state;
298 nvlist_t *nvinfo;
299
300 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
301 mmp_state = fnvlist_lookup_uint64(nvinfo,
302 ZPOOL_CONFIG_MMP_STATE);
303
304 if (mmp_state == MMP_STATE_ACTIVE)
305 return (ZPOOL_STATUS_HOSTID_ACTIVE);
306 else if (mmp_state == MMP_STATE_NO_HOSTID)
307 return (ZPOOL_STATUS_HOSTID_REQUIRED);
308 else
309 return (ZPOOL_STATUS_HOSTID_MISMATCH);
310 }
311
312 /*
313 * Pool last accessed by another system.
314 */
315 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
316 if (hostid != 0 && (unsigned long)hostid != system_hostid &&
317 stateval == POOL_STATE_ACTIVE)
318 return (ZPOOL_STATUS_HOSTID_MISMATCH);
319
320 /*
321 * Newer on-disk version.
322 */
323 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
324 vs->vs_aux == VDEV_AUX_VERSION_NEWER)
325 return (ZPOOL_STATUS_VERSION_NEWER);
326
327 /*
328 * Unsupported feature(s).
329 */
330 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
331 vs->vs_aux == VDEV_AUX_UNSUP_FEAT) {
332 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config,
333 ZPOOL_CONFIG_LOAD_INFO);
334 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_CAN_RDONLY))
335 return (ZPOOL_STATUS_UNSUP_FEAT_WRITE);
336 return (ZPOOL_STATUS_UNSUP_FEAT_READ);
337 }
338
339 /*
340 * Check that the config is complete.
341 */
342 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
343 vs->vs_aux == VDEV_AUX_BAD_GUID_SUM)
344 return (ZPOOL_STATUS_BAD_GUID_SUM);
345
346 /*
347 * Check whether the pool has suspended.
348 */
349 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED,
350 &suspended) == 0) {
351 uint64_t reason;
352
353 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED_REASON,
354 &reason) == 0 && reason == ZIO_SUSPEND_MMP)
355 return (ZPOOL_STATUS_IO_FAILURE_MMP);
356
357 if (suspended == ZIO_FAILURE_MODE_CONTINUE)
358 return (ZPOOL_STATUS_IO_FAILURE_CONTINUE);
359 return (ZPOOL_STATUS_IO_FAILURE_WAIT);
360 }
361
362 /*
363 * Could not read a log.
364 */
365 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
366 vs->vs_aux == VDEV_AUX_BAD_LOG) {
367 return (ZPOOL_STATUS_BAD_LOG);
368 }
369
370 /*
371 * Bad devices in non-replicated config.
372 */
373 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
374 find_vdev_problem(nvroot, vdev_faulted, B_TRUE))
375 return (ZPOOL_STATUS_FAULTED_DEV_NR);
376
377 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
378 find_vdev_problem(nvroot, vdev_missing, B_TRUE))
379 return (ZPOOL_STATUS_MISSING_DEV_NR);
380
381 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
382 find_vdev_problem(nvroot, vdev_broken, B_TRUE))
383 return (ZPOOL_STATUS_CORRUPT_LABEL_NR);
384
385 /*
386 * Corrupted pool metadata
387 */
388 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
389 vs->vs_aux == VDEV_AUX_CORRUPT_DATA)
390 return (ZPOOL_STATUS_CORRUPT_POOL);
391
392 /*
393 * Persistent data errors.
394 */
395 if (!isimport) {
396 uint64_t nerr;
397 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
398 &nerr) == 0 && nerr != 0)
399 return (ZPOOL_STATUS_CORRUPT_DATA);
400 }
401
402 /*
403 * Missing devices in a replicated config.
404 */
405 if (find_vdev_problem(nvroot, vdev_faulted, B_TRUE))
406 return (ZPOOL_STATUS_FAULTED_DEV_R);
407 if (find_vdev_problem(nvroot, vdev_missing, B_TRUE))
408 return (ZPOOL_STATUS_MISSING_DEV_R);
409 if (find_vdev_problem(nvroot, vdev_broken, B_TRUE))
410 return (ZPOOL_STATUS_CORRUPT_LABEL_R);
411
412 /*
413 * Devices with errors
414 */
415 if (!isimport && find_vdev_problem(nvroot, vdev_errors, B_TRUE))
416 return (ZPOOL_STATUS_FAILING_DEV);
417
418 /*
419 * Offlined devices
420 */
421 if (find_vdev_problem(nvroot, vdev_offlined, B_TRUE))
422 return (ZPOOL_STATUS_OFFLINE_DEV);
423
424 /*
425 * Removed device
426 */
427 if (find_vdev_problem(nvroot, vdev_removed, B_TRUE))
428 return (ZPOOL_STATUS_REMOVED_DEV);
429
430 /*
431 * Suboptimal, but usable, ashift configuration.
432 */
433 if (find_vdev_problem(nvroot, vdev_non_native_ashift, B_FALSE))
434 return (ZPOOL_STATUS_NON_NATIVE_ASHIFT);
435
436 /*
437 * Informational errata available.
438 */
439 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRATA, &errata);
440 if (errata) {
441 *erratap = errata;
442 return (ZPOOL_STATUS_ERRATA);
443 }
444
445 /*
446 * Outdated, but usable, version
447 */
448 if (SPA_VERSION_IS_SUPPORTED(version) && version != SPA_VERSION) {
449 /* "legacy" compatibility disables old version reporting */
450 if (compat != NULL && strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0)
451 return (ZPOOL_STATUS_OK);
452 else
453 return (ZPOOL_STATUS_VERSION_OLDER);
454 }
455
456 /*
457 * Usable pool with disabled or superfluous features
458 * (superfluous = beyond what's requested by 'compatibility')
459 */
460 if (version >= SPA_VERSION_FEATURES) {
461 int i;
462 nvlist_t *feat;
463
464 if (isimport) {
465 feat = fnvlist_lookup_nvlist(config,
466 ZPOOL_CONFIG_LOAD_INFO);
467 if (nvlist_exists(feat, ZPOOL_CONFIG_ENABLED_FEAT))
468 feat = fnvlist_lookup_nvlist(feat,
469 ZPOOL_CONFIG_ENABLED_FEAT);
470 } else {
471 feat = fnvlist_lookup_nvlist(config,
472 ZPOOL_CONFIG_FEATURE_STATS);
473 }
474
475 /* check against all features, or limited set? */
476 boolean_t c_features[SPA_FEATURES];
477
478 switch (zpool_load_compat(compat, c_features, NULL, 0)) {
479 case ZPOOL_COMPATIBILITY_OK:
480 case ZPOOL_COMPATIBILITY_WARNTOKEN:
481 break;
482 default:
483 return (ZPOOL_STATUS_COMPATIBILITY_ERR);
484 }
485 for (i = 0; i < SPA_FEATURES; i++) {
486 zfeature_info_t *fi = &spa_feature_table[i];
487 if (!fi->fi_zfs_mod_supported)
488 continue;
489 if (c_features[i] && !nvlist_exists(feat, fi->fi_guid))
490 return (ZPOOL_STATUS_FEAT_DISABLED);
491 if (!c_features[i] && nvlist_exists(feat, fi->fi_guid))
492 return (ZPOOL_STATUS_INCOMPATIBLE_FEAT);
493 }
494 }
495
496 return (ZPOOL_STATUS_OK);
497 }
498
499 zpool_status_t
zpool_get_status(zpool_handle_t * zhp,const char ** msgid,zpool_errata_t * errata)500 zpool_get_status(zpool_handle_t *zhp, const char **msgid,
501 zpool_errata_t *errata)
502 {
503 /*
504 * pass in the desired feature set, as
505 * it affects check for disabled features
506 */
507 char compatibility[ZFS_MAXPROPLEN];
508 if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compatibility,
509 ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
510 compatibility[0] = '\0';
511
512 zpool_status_t ret = check_status(zhp->zpool_config, B_FALSE, errata,
513 compatibility);
514
515 if (msgid != NULL) {
516 if (ret >= NMSGID)
517 *msgid = NULL;
518 else
519 *msgid = zfs_msgid_table[ret];
520 }
521 return (ret);
522 }
523
524 zpool_status_t
zpool_import_status(nvlist_t * config,const char ** msgid,zpool_errata_t * errata)525 zpool_import_status(nvlist_t *config, const char **msgid,
526 zpool_errata_t *errata)
527 {
528 zpool_status_t ret = check_status(config, B_TRUE, errata, NULL);
529
530 if (ret >= NMSGID)
531 *msgid = NULL;
532 else
533 *msgid = zfs_msgid_table[ret];
534
535 return (ret);
536 }
537