1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2012 by Delphix. All rights reserved.
26 * Copyright (c) 2013 Steven Hartland. All rights reserved.
27 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
28 */
29
30 /*
31 * This file contains the functions which analyze the status of a pool. This
32 * include both the status of an active pool, as well as the status exported
33 * pools. Returns one of the ZPOOL_STATUS_* defines describing the status of
34 * the pool. This status is independent (to a certain degree) from the state of
35 * the pool. A pool's state describes only whether or not it is capable of
36 * providing the necessary fault tolerance for data. The status describes the
37 * overall status of devices. A pool that is online can still have a device
38 * that is experiencing errors.
39 *
40 * Only a subset of the possible faults can be detected using 'zpool status',
41 * and not all possible errors correspond to a FMA message ID. The explanation
42 * is left up to the caller, depending on whether it is a live pool or an
43 * import.
44 */
45
46 #include <libzfs.h>
47 #include <libzutil.h>
48 #include <stdlib.h>
49 #include <string.h>
50 #include <unistd.h>
51 #include <sys/systeminfo.h>
52 #include "libzfs_impl.h"
53 #include "zfeature_common.h"
54
55 /*
56 * Message ID table. This must be kept in sync with the ZPOOL_STATUS_* defines
57 * in include/libzfs.h. Note that there are some status results which go past
58 * the end of this table, and hence have no associated message ID.
59 */
60 static const char *const zfs_msgid_table[] = {
61 "ZFS-8000-14", /* ZPOOL_STATUS_CORRUPT_CACHE */
62 "ZFS-8000-2Q", /* ZPOOL_STATUS_MISSING_DEV_R */
63 "ZFS-8000-3C", /* ZPOOL_STATUS_MISSING_DEV_NR */
64 "ZFS-8000-4J", /* ZPOOL_STATUS_CORRUPT_LABEL_R */
65 "ZFS-8000-5E", /* ZPOOL_STATUS_CORRUPT_LABEL_NR */
66 "ZFS-8000-6X", /* ZPOOL_STATUS_BAD_GUID_SUM */
67 "ZFS-8000-72", /* ZPOOL_STATUS_CORRUPT_POOL */
68 "ZFS-8000-8A", /* ZPOOL_STATUS_CORRUPT_DATA */
69 "ZFS-8000-9P", /* ZPOOL_STATUS_FAILING_DEV */
70 "ZFS-8000-A5", /* ZPOOL_STATUS_VERSION_NEWER */
71 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_MISMATCH */
72 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_ACTIVE */
73 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_REQUIRED */
74 "ZFS-8000-HC", /* ZPOOL_STATUS_IO_FAILURE_WAIT */
75 "ZFS-8000-JQ", /* ZPOOL_STATUS_IO_FAILURE_CONTINUE */
76 "ZFS-8000-MM", /* ZPOOL_STATUS_IO_FAILURE_MMP */
77 "ZFS-8000-K4", /* ZPOOL_STATUS_BAD_LOG */
78 "ZFS-8000-ER", /* ZPOOL_STATUS_ERRATA */
79 /*
80 * The following results have no message ID.
81 * ZPOOL_STATUS_UNSUP_FEAT_READ
82 * ZPOOL_STATUS_UNSUP_FEAT_WRITE
83 * ZPOOL_STATUS_FAULTED_DEV_R
84 * ZPOOL_STATUS_FAULTED_DEV_NR
85 * ZPOOL_STATUS_VERSION_OLDER
86 * ZPOOL_STATUS_FEAT_DISABLED
87 * ZPOOL_STATUS_RESILVERING
88 * ZPOOL_STATUS_OFFLINE_DEV
89 * ZPOOL_STATUS_REMOVED_DEV
90 * ZPOOL_STATUS_REBUILDING
91 * ZPOOL_STATUS_REBUILD_SCRUB
92 * ZPOOL_STATUS_COMPATIBILITY_ERR
93 * ZPOOL_STATUS_INCOMPATIBLE_FEAT
94 * ZPOOL_STATUS_OK
95 */
96 };
97
98 #define NMSGID (sizeof (zfs_msgid_table) / sizeof (zfs_msgid_table[0]))
99
100 static int
vdev_missing(vdev_stat_t * vs,uint_t vsc,void * arg)101 vdev_missing(vdev_stat_t *vs, uint_t vsc, void *arg)
102 {
103 (void) vsc, (void) arg;
104 return (vs->vs_state == VDEV_STATE_CANT_OPEN &&
105 vs->vs_aux == VDEV_AUX_OPEN_FAILED);
106 }
107
108 static int
vdev_faulted(vdev_stat_t * vs,uint_t vsc,void * arg)109 vdev_faulted(vdev_stat_t *vs, uint_t vsc, void *arg)
110 {
111 (void) vsc, (void) arg;
112 return (vs->vs_state == VDEV_STATE_FAULTED);
113 }
114
115 static int
vdev_errors(vdev_stat_t * vs,uint_t vsc,void * arg)116 vdev_errors(vdev_stat_t *vs, uint_t vsc, void *arg)
117 {
118 (void) vsc, (void) arg;
119 return (vs->vs_state == VDEV_STATE_DEGRADED ||
120 vs->vs_read_errors != 0 || vs->vs_write_errors != 0 ||
121 vs->vs_checksum_errors != 0);
122 }
123
124 static int
vdev_broken(vdev_stat_t * vs,uint_t vsc,void * arg)125 vdev_broken(vdev_stat_t *vs, uint_t vsc, void *arg)
126 {
127 (void) vsc, (void) arg;
128 return (vs->vs_state == VDEV_STATE_CANT_OPEN);
129 }
130
131 static int
vdev_offlined(vdev_stat_t * vs,uint_t vsc,void * arg)132 vdev_offlined(vdev_stat_t *vs, uint_t vsc, void *arg)
133 {
134 (void) vsc, (void) arg;
135 return (vs->vs_state == VDEV_STATE_OFFLINE);
136 }
137
138 static int
vdev_removed(vdev_stat_t * vs,uint_t vsc,void * arg)139 vdev_removed(vdev_stat_t *vs, uint_t vsc, void *arg)
140 {
141 (void) vsc, (void) arg;
142 return (vs->vs_state == VDEV_STATE_REMOVED);
143 }
144
145 static int
vdev_non_native_ashift(vdev_stat_t * vs,uint_t vsc,void * arg)146 vdev_non_native_ashift(vdev_stat_t *vs, uint_t vsc, void *arg)
147 {
148 uint64_t ashift = *(uint64_t *)arg;
149
150 return (VDEV_STAT_VALID(vs_physical_ashift, vsc) &&
151 (ashift == 0 || vs->vs_configured_ashift < ashift) &&
152 vs->vs_configured_ashift < vs->vs_physical_ashift);
153 }
154
155 /*
156 * Detect if any leaf devices that have seen errors or could not be opened.
157 */
158 static boolean_t
find_vdev_problem(nvlist_t * vdev,int (* func)(vdev_stat_t *,uint_t,void *),void * arg,boolean_t ignore_replacing)159 find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t, void *),
160 void *arg, boolean_t ignore_replacing)
161 {
162 nvlist_t **child;
163 uint_t c, children;
164
165 /*
166 * Ignore problems within a 'replacing' vdev, since we're presumably in
167 * the process of repairing any such errors, and don't want to call them
168 * out again. We'll pick up the fact that a resilver is happening
169 * later.
170 */
171 if (ignore_replacing == B_TRUE) {
172 const char *type = fnvlist_lookup_string(vdev,
173 ZPOOL_CONFIG_TYPE);
174 if (strcmp(type, VDEV_TYPE_REPLACING) == 0)
175 return (B_FALSE);
176 }
177
178 if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_CHILDREN, &child,
179 &children) == 0) {
180 for (c = 0; c < children; c++) {
181 if (find_vdev_problem(child[c], func, arg,
182 ignore_replacing))
183 return (B_TRUE);
184 }
185 } else {
186 uint_t vsc;
187 vdev_stat_t *vs = (vdev_stat_t *)fnvlist_lookup_uint64_array(
188 vdev, ZPOOL_CONFIG_VDEV_STATS, &vsc);
189 if (func(vs, vsc, arg) != 0)
190 return (B_TRUE);
191 }
192
193 /*
194 * Check any L2 cache devs
195 */
196 if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_L2CACHE, &child,
197 &children) == 0) {
198 for (c = 0; c < children; c++) {
199 if (find_vdev_problem(child[c], func, arg,
200 ignore_replacing))
201 return (B_TRUE);
202 }
203 }
204
205 return (B_FALSE);
206 }
207
208 /*
209 * Active pool health status.
210 *
211 * To determine the status for a pool, we make several passes over the config,
212 * picking the most egregious error we find. In order of importance, we do the
213 * following:
214 *
215 * - Check for a complete and valid configuration
216 * - Look for any faulted or missing devices in a non-replicated config
217 * - Check for any data errors
218 * - Check for any faulted or missing devices in a replicated config
219 * - Look for any devices showing errors
220 * - Check for any resilvering or rebuilding devices
221 *
222 * There can obviously be multiple errors within a single pool, so this routine
223 * only picks the most damaging of all the current errors to report.
224 */
225 static zpool_status_t
check_status(nvlist_t * config,boolean_t isimport,zpool_errata_t * erratap,const char * compat,uint64_t ashift)226 check_status(nvlist_t *config, boolean_t isimport,
227 zpool_errata_t *erratap, const char *compat, uint64_t ashift)
228 {
229 pool_scan_stat_t *ps = NULL;
230 uint_t vsc, psc;
231 uint64_t suspended;
232 uint64_t hostid = 0;
233 uint64_t errata = 0;
234 unsigned long system_hostid = get_system_hostid();
235
236 uint64_t version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
237 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
238 ZPOOL_CONFIG_VDEV_TREE);
239 vdev_stat_t *vs = (vdev_stat_t *)fnvlist_lookup_uint64_array(nvroot,
240 ZPOOL_CONFIG_VDEV_STATS, &vsc);
241 uint64_t stateval = fnvlist_lookup_uint64(config,
242 ZPOOL_CONFIG_POOL_STATE);
243
244 /*
245 * Currently resilvering a vdev
246 */
247 (void) nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
248 (uint64_t **)&ps, &psc);
249 if (ps != NULL && ps->pss_func == POOL_SCAN_RESILVER &&
250 ps->pss_state == DSS_SCANNING)
251 return (ZPOOL_STATUS_RESILVERING);
252
253 /*
254 * Currently rebuilding a vdev, check top-level vdevs.
255 */
256 vdev_rebuild_stat_t *vrs = NULL;
257 nvlist_t **child;
258 uint_t c, i, children;
259 uint64_t rebuild_end_time = 0;
260 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
261 &child, &children) == 0) {
262 for (c = 0; c < children; c++) {
263 if ((nvlist_lookup_uint64_array(child[c],
264 ZPOOL_CONFIG_REBUILD_STATS,
265 (uint64_t **)&vrs, &i) == 0) && (vrs != NULL)) {
266 uint64_t state = vrs->vrs_state;
267
268 if (state == VDEV_REBUILD_ACTIVE) {
269 return (ZPOOL_STATUS_REBUILDING);
270 } else if (state == VDEV_REBUILD_COMPLETE &&
271 vrs->vrs_end_time > rebuild_end_time) {
272 rebuild_end_time = vrs->vrs_end_time;
273 }
274 }
275 }
276
277 /*
278 * If we can determine when the last scrub was run, and it
279 * was before the last rebuild completed, then recommend
280 * that the pool be scrubbed to verify all checksums. When
281 * ps is NULL we can infer the pool has never been scrubbed.
282 */
283 if (rebuild_end_time > 0) {
284 if (ps != NULL) {
285 if ((ps->pss_state == DSS_FINISHED &&
286 ps->pss_func == POOL_SCAN_SCRUB &&
287 rebuild_end_time > ps->pss_end_time) ||
288 ps->pss_state == DSS_NONE)
289 return (ZPOOL_STATUS_REBUILD_SCRUB);
290 } else {
291 return (ZPOOL_STATUS_REBUILD_SCRUB);
292 }
293 }
294 }
295
296 /*
297 * The multihost property is set and the pool may be active.
298 */
299 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
300 vs->vs_aux == VDEV_AUX_ACTIVE) {
301 mmp_state_t mmp_state;
302 nvlist_t *nvinfo;
303
304 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
305 mmp_state = fnvlist_lookup_uint64(nvinfo,
306 ZPOOL_CONFIG_MMP_STATE);
307
308 if (mmp_state == MMP_STATE_ACTIVE)
309 return (ZPOOL_STATUS_HOSTID_ACTIVE);
310 else if (mmp_state == MMP_STATE_NO_HOSTID)
311 return (ZPOOL_STATUS_HOSTID_REQUIRED);
312 else
313 return (ZPOOL_STATUS_HOSTID_MISMATCH);
314 }
315
316 /*
317 * Pool last accessed by another system.
318 */
319 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
320 if (hostid != 0 && (unsigned long)hostid != system_hostid &&
321 stateval == POOL_STATE_ACTIVE)
322 return (ZPOOL_STATUS_HOSTID_MISMATCH);
323
324 /*
325 * Newer on-disk version.
326 */
327 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
328 vs->vs_aux == VDEV_AUX_VERSION_NEWER)
329 return (ZPOOL_STATUS_VERSION_NEWER);
330
331 /*
332 * Unsupported feature(s).
333 */
334 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
335 vs->vs_aux == VDEV_AUX_UNSUP_FEAT) {
336 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config,
337 ZPOOL_CONFIG_LOAD_INFO);
338 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_CAN_RDONLY))
339 return (ZPOOL_STATUS_UNSUP_FEAT_WRITE);
340 return (ZPOOL_STATUS_UNSUP_FEAT_READ);
341 }
342
343 /*
344 * Check that the config is complete.
345 */
346 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
347 vs->vs_aux == VDEV_AUX_BAD_GUID_SUM)
348 return (ZPOOL_STATUS_BAD_GUID_SUM);
349
350 /*
351 * Check whether the pool has suspended.
352 */
353 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED,
354 &suspended) == 0) {
355 uint64_t reason;
356
357 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED_REASON,
358 &reason) == 0 && reason == ZIO_SUSPEND_MMP)
359 return (ZPOOL_STATUS_IO_FAILURE_MMP);
360
361 if (suspended == ZIO_FAILURE_MODE_CONTINUE)
362 return (ZPOOL_STATUS_IO_FAILURE_CONTINUE);
363 return (ZPOOL_STATUS_IO_FAILURE_WAIT);
364 }
365
366 /*
367 * Could not read a log.
368 */
369 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
370 vs->vs_aux == VDEV_AUX_BAD_LOG) {
371 return (ZPOOL_STATUS_BAD_LOG);
372 }
373
374 /*
375 * Bad devices in non-replicated config.
376 */
377 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
378 find_vdev_problem(nvroot, vdev_faulted, NULL, B_TRUE))
379 return (ZPOOL_STATUS_FAULTED_DEV_NR);
380
381 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
382 find_vdev_problem(nvroot, vdev_missing, NULL, B_TRUE))
383 return (ZPOOL_STATUS_MISSING_DEV_NR);
384
385 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
386 find_vdev_problem(nvroot, vdev_broken, NULL, B_TRUE))
387 return (ZPOOL_STATUS_CORRUPT_LABEL_NR);
388
389 /*
390 * Corrupted pool metadata
391 */
392 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
393 vs->vs_aux == VDEV_AUX_CORRUPT_DATA)
394 return (ZPOOL_STATUS_CORRUPT_POOL);
395
396 /*
397 * Persistent data errors.
398 */
399 if (!isimport) {
400 uint64_t nerr;
401 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
402 &nerr) == 0 && nerr != 0)
403 return (ZPOOL_STATUS_CORRUPT_DATA);
404 }
405
406 /*
407 * Missing devices in a replicated config.
408 */
409 if (find_vdev_problem(nvroot, vdev_faulted, NULL, B_TRUE))
410 return (ZPOOL_STATUS_FAULTED_DEV_R);
411 if (find_vdev_problem(nvroot, vdev_missing, NULL, B_TRUE))
412 return (ZPOOL_STATUS_MISSING_DEV_R);
413 if (find_vdev_problem(nvroot, vdev_broken, NULL, B_TRUE))
414 return (ZPOOL_STATUS_CORRUPT_LABEL_R);
415
416 /*
417 * Devices with errors
418 */
419 if (!isimport && find_vdev_problem(nvroot, vdev_errors, NULL, B_TRUE))
420 return (ZPOOL_STATUS_FAILING_DEV);
421
422 /*
423 * Offlined devices
424 */
425 if (find_vdev_problem(nvroot, vdev_offlined, NULL, B_TRUE))
426 return (ZPOOL_STATUS_OFFLINE_DEV);
427
428 /*
429 * Removed device
430 */
431 if (find_vdev_problem(nvroot, vdev_removed, NULL, B_TRUE))
432 return (ZPOOL_STATUS_REMOVED_DEV);
433
434 /*
435 * Suboptimal, but usable, ashift configuration.
436 */
437 if (!isimport &&
438 getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") == NULL &&
439 find_vdev_problem(nvroot, vdev_non_native_ashift, &ashift, B_FALSE))
440 return (ZPOOL_STATUS_NON_NATIVE_ASHIFT);
441
442 /*
443 * Informational errata available.
444 */
445 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRATA, &errata);
446 if (errata) {
447 *erratap = errata;
448 return (ZPOOL_STATUS_ERRATA);
449 }
450
451 /*
452 * Outdated, but usable, version
453 */
454 if (SPA_VERSION_IS_SUPPORTED(version) && version != SPA_VERSION) {
455 /* "legacy" compatibility disables old version reporting */
456 if (compat != NULL && strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0)
457 return (ZPOOL_STATUS_OK);
458 else
459 return (ZPOOL_STATUS_VERSION_OLDER);
460 }
461
462 /*
463 * Usable pool with disabled or superfluous features
464 * (superfluous = beyond what's requested by 'compatibility')
465 */
466 if (version >= SPA_VERSION_FEATURES) {
467 int i;
468 nvlist_t *feat;
469
470 if (isimport) {
471 feat = fnvlist_lookup_nvlist(config,
472 ZPOOL_CONFIG_LOAD_INFO);
473 if (nvlist_exists(feat, ZPOOL_CONFIG_ENABLED_FEAT))
474 feat = fnvlist_lookup_nvlist(feat,
475 ZPOOL_CONFIG_ENABLED_FEAT);
476 } else {
477 feat = fnvlist_lookup_nvlist(config,
478 ZPOOL_CONFIG_FEATURE_STATS);
479 }
480
481 /* check against all features, or limited set? */
482 boolean_t c_features[SPA_FEATURES];
483
484 switch (zpool_load_compat(compat, c_features, NULL, 0)) {
485 case ZPOOL_COMPATIBILITY_OK:
486 case ZPOOL_COMPATIBILITY_WARNTOKEN:
487 break;
488 default:
489 return (ZPOOL_STATUS_COMPATIBILITY_ERR);
490 }
491 for (i = 0; i < SPA_FEATURES; i++) {
492 zfeature_info_t *fi = &spa_feature_table[i];
493 if (!fi->fi_zfs_mod_supported ||
494 (fi->fi_flags & ZFEATURE_FLAG_NO_UPGRADE))
495 continue;
496 if (c_features[i] && !nvlist_exists(feat, fi->fi_guid))
497 return (ZPOOL_STATUS_FEAT_DISABLED);
498 if (!c_features[i] && nvlist_exists(feat, fi->fi_guid))
499 return (ZPOOL_STATUS_INCOMPATIBLE_FEAT);
500 }
501 }
502
503 return (ZPOOL_STATUS_OK);
504 }
505
506 zpool_status_t
zpool_get_status(zpool_handle_t * zhp,const char ** msgid,zpool_errata_t * errata)507 zpool_get_status(zpool_handle_t *zhp, const char **msgid,
508 zpool_errata_t *errata)
509 {
510 /*
511 * pass in the desired feature set, as
512 * it affects check for disabled features
513 */
514 char compatibility[ZFS_MAXPROPLEN];
515 if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compatibility,
516 ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
517 compatibility[0] = '\0';
518
519 uint64_t ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, NULL);
520
521 zpool_status_t ret = check_status(zhp->zpool_config, B_FALSE, errata,
522 compatibility, ashift);
523
524 if (msgid != NULL) {
525 if (ret >= NMSGID)
526 *msgid = NULL;
527 else
528 *msgid = zfs_msgid_table[ret];
529 }
530 return (ret);
531 }
532
533 zpool_status_t
zpool_import_status(nvlist_t * config,const char ** msgid,zpool_errata_t * errata)534 zpool_import_status(nvlist_t *config, const char **msgid,
535 zpool_errata_t *errata)
536 {
537 zpool_status_t ret = check_status(config, B_TRUE, errata, NULL, 0);
538
539 if (ret >= NMSGID)
540 *msgid = NULL;
541 else
542 *msgid = zfs_msgid_table[ret];
543
544 return (ret);
545 }
546