1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 25 * Copyright (c) 2012 by Delphix. All rights reserved. 26 * Copyright (c) 2013 Steven Hartland. All rights reserved. 27 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 28 */ 29 30 /* 31 * This file contains the functions which analyze the status of a pool. This 32 * include both the status of an active pool, as well as the status exported 33 * pools. Returns one of the ZPOOL_STATUS_* defines describing the status of 34 * the pool. This status is independent (to a certain degree) from the state of 35 * the pool. A pool's state describes only whether or not it is capable of 36 * providing the necessary fault tolerance for data. The status describes the 37 * overall status of devices. A pool that is online can still have a device 38 * that is experiencing errors. 39 * 40 * Only a subset of the possible faults can be detected using 'zpool status', 41 * and not all possible errors correspond to a FMA message ID. The explanation 42 * is left up to the caller, depending on whether it is a live pool or an 43 * import. 44 */ 45 46 #include <libzfs.h> 47 #include <libzutil.h> 48 #include <stdlib.h> 49 #include <string.h> 50 #include <unistd.h> 51 #include <sys/systeminfo.h> 52 #include "libzfs_impl.h" 53 #include "zfeature_common.h" 54 55 /* 56 * Message ID table. This must be kept in sync with the ZPOOL_STATUS_* defines 57 * in include/libzfs.h. Note that there are some status results which go past 58 * the end of this table, and hence have no associated message ID. 59 */ 60 static const char *const zfs_msgid_table[] = { 61 "ZFS-8000-14", /* ZPOOL_STATUS_CORRUPT_CACHE */ 62 "ZFS-8000-2Q", /* ZPOOL_STATUS_MISSING_DEV_R */ 63 "ZFS-8000-3C", /* ZPOOL_STATUS_MISSING_DEV_NR */ 64 "ZFS-8000-4J", /* ZPOOL_STATUS_CORRUPT_LABEL_R */ 65 "ZFS-8000-5E", /* ZPOOL_STATUS_CORRUPT_LABEL_NR */ 66 "ZFS-8000-6X", /* ZPOOL_STATUS_BAD_GUID_SUM */ 67 "ZFS-8000-72", /* ZPOOL_STATUS_CORRUPT_POOL */ 68 "ZFS-8000-8A", /* ZPOOL_STATUS_CORRUPT_DATA */ 69 "ZFS-8000-9P", /* ZPOOL_STATUS_FAILING_DEV */ 70 "ZFS-8000-A5", /* ZPOOL_STATUS_VERSION_NEWER */ 71 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_MISMATCH */ 72 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_ACTIVE */ 73 "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_REQUIRED */ 74 "ZFS-8000-HC", /* ZPOOL_STATUS_IO_FAILURE_WAIT */ 75 "ZFS-8000-JQ", /* ZPOOL_STATUS_IO_FAILURE_CONTINUE */ 76 "ZFS-8000-MM", /* ZPOOL_STATUS_IO_FAILURE_MMP */ 77 "ZFS-8000-K4", /* ZPOOL_STATUS_BAD_LOG */ 78 "ZFS-8000-ER", /* ZPOOL_STATUS_ERRATA */ 79 /* 80 * The following results have no message ID. 81 * ZPOOL_STATUS_UNSUP_FEAT_READ 82 * ZPOOL_STATUS_UNSUP_FEAT_WRITE 83 * ZPOOL_STATUS_FAULTED_DEV_R 84 * ZPOOL_STATUS_FAULTED_DEV_NR 85 * ZPOOL_STATUS_VERSION_OLDER 86 * ZPOOL_STATUS_FEAT_DISABLED 87 * ZPOOL_STATUS_RESILVERING 88 * ZPOOL_STATUS_OFFLINE_DEV 89 * ZPOOL_STATUS_REMOVED_DEV 90 * ZPOOL_STATUS_REBUILDING 91 * ZPOOL_STATUS_REBUILD_SCRUB 92 * ZPOOL_STATUS_COMPATIBILITY_ERR 93 * ZPOOL_STATUS_INCOMPATIBLE_FEAT 94 * ZPOOL_STATUS_OK 95 */ 96 }; 97 98 #define NMSGID (sizeof (zfs_msgid_table) / sizeof (zfs_msgid_table[0])) 99 100 static int 101 vdev_missing(vdev_stat_t *vs, uint_t vsc, void *arg) 102 { 103 (void) vsc, (void) arg; 104 return (vs->vs_state == VDEV_STATE_CANT_OPEN && 105 vs->vs_aux == VDEV_AUX_OPEN_FAILED); 106 } 107 108 static int 109 vdev_faulted(vdev_stat_t *vs, uint_t vsc, void *arg) 110 { 111 (void) vsc, (void) arg; 112 return (vs->vs_state == VDEV_STATE_FAULTED); 113 } 114 115 static int 116 vdev_errors(vdev_stat_t *vs, uint_t vsc, void *arg) 117 { 118 (void) vsc, (void) arg; 119 return (vs->vs_state == VDEV_STATE_DEGRADED || 120 vs->vs_read_errors != 0 || vs->vs_write_errors != 0 || 121 vs->vs_checksum_errors != 0); 122 } 123 124 static int 125 vdev_broken(vdev_stat_t *vs, uint_t vsc, void *arg) 126 { 127 (void) vsc, (void) arg; 128 return (vs->vs_state == VDEV_STATE_CANT_OPEN); 129 } 130 131 static int 132 vdev_offlined(vdev_stat_t *vs, uint_t vsc, void *arg) 133 { 134 (void) vsc, (void) arg; 135 return (vs->vs_state == VDEV_STATE_OFFLINE); 136 } 137 138 static int 139 vdev_removed(vdev_stat_t *vs, uint_t vsc, void *arg) 140 { 141 (void) vsc, (void) arg; 142 return (vs->vs_state == VDEV_STATE_REMOVED); 143 } 144 145 static int 146 vdev_non_native_ashift(vdev_stat_t *vs, uint_t vsc, void *arg) 147 { 148 uint64_t ashift = *(uint64_t *)arg; 149 150 return (VDEV_STAT_VALID(vs_physical_ashift, vsc) && 151 (ashift == 0 || vs->vs_configured_ashift < ashift) && 152 vs->vs_configured_ashift < vs->vs_physical_ashift); 153 } 154 155 /* 156 * Detect if any leaf devices that have seen errors or could not be opened. 157 */ 158 static boolean_t 159 find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t, void *), 160 void *arg, boolean_t ignore_replacing) 161 { 162 nvlist_t **child; 163 uint_t c, children; 164 165 /* 166 * Ignore problems within a 'replacing' vdev, since we're presumably in 167 * the process of repairing any such errors, and don't want to call them 168 * out again. We'll pick up the fact that a resilver is happening 169 * later. 170 */ 171 if (ignore_replacing == B_TRUE) { 172 const char *type = fnvlist_lookup_string(vdev, 173 ZPOOL_CONFIG_TYPE); 174 if (strcmp(type, VDEV_TYPE_REPLACING) == 0) 175 return (B_FALSE); 176 } 177 178 if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_CHILDREN, &child, 179 &children) == 0) { 180 for (c = 0; c < children; c++) { 181 if (find_vdev_problem(child[c], func, arg, 182 ignore_replacing)) 183 return (B_TRUE); 184 } 185 } else { 186 uint_t vsc; 187 vdev_stat_t *vs = (vdev_stat_t *)fnvlist_lookup_uint64_array( 188 vdev, ZPOOL_CONFIG_VDEV_STATS, &vsc); 189 if (func(vs, vsc, arg) != 0) 190 return (B_TRUE); 191 } 192 193 /* 194 * Check any L2 cache devs 195 */ 196 if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_L2CACHE, &child, 197 &children) == 0) { 198 for (c = 0; c < children; c++) { 199 if (find_vdev_problem(child[c], func, arg, 200 ignore_replacing)) 201 return (B_TRUE); 202 } 203 } 204 205 return (B_FALSE); 206 } 207 208 /* 209 * Active pool health status. 210 * 211 * To determine the status for a pool, we make several passes over the config, 212 * picking the most egregious error we find. In order of importance, we do the 213 * following: 214 * 215 * - Check for a complete and valid configuration 216 * - Look for any faulted or missing devices in a non-replicated config 217 * - Check for any data errors 218 * - Check for any faulted or missing devices in a replicated config 219 * - Look for any devices showing errors 220 * - Check for any resilvering or rebuilding devices 221 * 222 * There can obviously be multiple errors within a single pool, so this routine 223 * only picks the most damaging of all the current errors to report. 224 */ 225 static zpool_status_t 226 check_status(nvlist_t *config, boolean_t isimport, 227 zpool_errata_t *erratap, const char *compat, uint64_t ashift) 228 { 229 pool_scan_stat_t *ps = NULL; 230 uint_t vsc, psc; 231 uint64_t suspended; 232 uint64_t hostid = 0; 233 uint64_t errata = 0; 234 unsigned long system_hostid = get_system_hostid(); 235 236 uint64_t version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION); 237 nvlist_t *nvroot = fnvlist_lookup_nvlist(config, 238 ZPOOL_CONFIG_VDEV_TREE); 239 vdev_stat_t *vs = (vdev_stat_t *)fnvlist_lookup_uint64_array(nvroot, 240 ZPOOL_CONFIG_VDEV_STATS, &vsc); 241 uint64_t stateval = fnvlist_lookup_uint64(config, 242 ZPOOL_CONFIG_POOL_STATE); 243 244 /* 245 * Currently resilvering a vdev 246 */ 247 (void) nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS, 248 (uint64_t **)&ps, &psc); 249 if (ps != NULL && ps->pss_func == POOL_SCAN_RESILVER && 250 ps->pss_state == DSS_SCANNING) 251 return (ZPOOL_STATUS_RESILVERING); 252 253 /* 254 * Currently rebuilding a vdev, check top-level vdevs. 255 */ 256 vdev_rebuild_stat_t *vrs = NULL; 257 nvlist_t **child; 258 uint_t c, i, children; 259 uint64_t rebuild_end_time = 0; 260 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, 261 &child, &children) == 0) { 262 for (c = 0; c < children; c++) { 263 if ((nvlist_lookup_uint64_array(child[c], 264 ZPOOL_CONFIG_REBUILD_STATS, 265 (uint64_t **)&vrs, &i) == 0) && (vrs != NULL)) { 266 uint64_t state = vrs->vrs_state; 267 268 if (state == VDEV_REBUILD_ACTIVE) { 269 return (ZPOOL_STATUS_REBUILDING); 270 } else if (state == VDEV_REBUILD_COMPLETE && 271 vrs->vrs_end_time > rebuild_end_time) { 272 rebuild_end_time = vrs->vrs_end_time; 273 } 274 } 275 } 276 277 /* 278 * If we can determine when the last scrub was run, and it 279 * was before the last rebuild completed, then recommend 280 * that the pool be scrubbed to verify all checksums. When 281 * ps is NULL we can infer the pool has never been scrubbed. 282 */ 283 if (rebuild_end_time > 0) { 284 if (ps != NULL) { 285 if ((ps->pss_state == DSS_FINISHED && 286 ps->pss_func == POOL_SCAN_SCRUB && 287 rebuild_end_time > ps->pss_end_time) || 288 ps->pss_state == DSS_NONE) 289 return (ZPOOL_STATUS_REBUILD_SCRUB); 290 } else { 291 return (ZPOOL_STATUS_REBUILD_SCRUB); 292 } 293 } 294 } 295 296 /* 297 * The multihost property is set and the pool may be active. 298 */ 299 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 300 vs->vs_aux == VDEV_AUX_ACTIVE) { 301 mmp_state_t mmp_state; 302 nvlist_t *nvinfo; 303 304 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); 305 mmp_state = fnvlist_lookup_uint64(nvinfo, 306 ZPOOL_CONFIG_MMP_STATE); 307 308 if (mmp_state == MMP_STATE_ACTIVE) 309 return (ZPOOL_STATUS_HOSTID_ACTIVE); 310 else if (mmp_state == MMP_STATE_NO_HOSTID) 311 return (ZPOOL_STATUS_HOSTID_REQUIRED); 312 else 313 return (ZPOOL_STATUS_HOSTID_MISMATCH); 314 } 315 316 /* 317 * Pool last accessed by another system. 318 */ 319 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid); 320 if (hostid != 0 && (unsigned long)hostid != system_hostid && 321 stateval == POOL_STATE_ACTIVE) 322 return (ZPOOL_STATUS_HOSTID_MISMATCH); 323 324 /* 325 * Newer on-disk version. 326 */ 327 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 328 vs->vs_aux == VDEV_AUX_VERSION_NEWER) 329 return (ZPOOL_STATUS_VERSION_NEWER); 330 331 /* 332 * Unsupported feature(s). 333 */ 334 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 335 vs->vs_aux == VDEV_AUX_UNSUP_FEAT) { 336 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, 337 ZPOOL_CONFIG_LOAD_INFO); 338 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_CAN_RDONLY)) 339 return (ZPOOL_STATUS_UNSUP_FEAT_WRITE); 340 return (ZPOOL_STATUS_UNSUP_FEAT_READ); 341 } 342 343 /* 344 * Check that the config is complete. 345 */ 346 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 347 vs->vs_aux == VDEV_AUX_BAD_GUID_SUM) 348 return (ZPOOL_STATUS_BAD_GUID_SUM); 349 350 /* 351 * Check whether the pool has suspended. 352 */ 353 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED, 354 &suspended) == 0) { 355 uint64_t reason; 356 357 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED_REASON, 358 &reason) == 0 && reason == ZIO_SUSPEND_MMP) 359 return (ZPOOL_STATUS_IO_FAILURE_MMP); 360 361 if (suspended == ZIO_FAILURE_MODE_CONTINUE) 362 return (ZPOOL_STATUS_IO_FAILURE_CONTINUE); 363 return (ZPOOL_STATUS_IO_FAILURE_WAIT); 364 } 365 366 /* 367 * Could not read a log. 368 */ 369 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 370 vs->vs_aux == VDEV_AUX_BAD_LOG) { 371 return (ZPOOL_STATUS_BAD_LOG); 372 } 373 374 /* 375 * Bad devices in non-replicated config. 376 */ 377 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 378 find_vdev_problem(nvroot, vdev_faulted, NULL, B_TRUE)) 379 return (ZPOOL_STATUS_FAULTED_DEV_NR); 380 381 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 382 find_vdev_problem(nvroot, vdev_missing, NULL, B_TRUE)) 383 return (ZPOOL_STATUS_MISSING_DEV_NR); 384 385 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 386 find_vdev_problem(nvroot, vdev_broken, NULL, B_TRUE)) 387 return (ZPOOL_STATUS_CORRUPT_LABEL_NR); 388 389 /* 390 * Corrupted pool metadata 391 */ 392 if (vs->vs_state == VDEV_STATE_CANT_OPEN && 393 vs->vs_aux == VDEV_AUX_CORRUPT_DATA) 394 return (ZPOOL_STATUS_CORRUPT_POOL); 395 396 /* 397 * Persistent data errors. 398 */ 399 if (!isimport) { 400 uint64_t nerr; 401 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT, 402 &nerr) == 0 && nerr != 0) 403 return (ZPOOL_STATUS_CORRUPT_DATA); 404 } 405 406 /* 407 * Missing devices in a replicated config. 408 */ 409 if (find_vdev_problem(nvroot, vdev_faulted, NULL, B_TRUE)) 410 return (ZPOOL_STATUS_FAULTED_DEV_R); 411 if (find_vdev_problem(nvroot, vdev_missing, NULL, B_TRUE)) 412 return (ZPOOL_STATUS_MISSING_DEV_R); 413 if (find_vdev_problem(nvroot, vdev_broken, NULL, B_TRUE)) 414 return (ZPOOL_STATUS_CORRUPT_LABEL_R); 415 416 /* 417 * Devices with errors 418 */ 419 if (!isimport && find_vdev_problem(nvroot, vdev_errors, NULL, B_TRUE)) 420 return (ZPOOL_STATUS_FAILING_DEV); 421 422 /* 423 * Offlined devices 424 */ 425 if (find_vdev_problem(nvroot, vdev_offlined, NULL, B_TRUE)) 426 return (ZPOOL_STATUS_OFFLINE_DEV); 427 428 /* 429 * Removed device 430 */ 431 if (find_vdev_problem(nvroot, vdev_removed, NULL, B_TRUE)) 432 return (ZPOOL_STATUS_REMOVED_DEV); 433 434 /* 435 * Suboptimal, but usable, ashift configuration. 436 */ 437 if (!isimport && 438 getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") == NULL && 439 find_vdev_problem(nvroot, vdev_non_native_ashift, &ashift, B_FALSE)) 440 return (ZPOOL_STATUS_NON_NATIVE_ASHIFT); 441 442 /* 443 * Informational errata available. 444 */ 445 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRATA, &errata); 446 if (errata) { 447 *erratap = errata; 448 return (ZPOOL_STATUS_ERRATA); 449 } 450 451 /* 452 * Outdated, but usable, version 453 */ 454 if (SPA_VERSION_IS_SUPPORTED(version) && version != SPA_VERSION) { 455 /* "legacy" compatibility disables old version reporting */ 456 if (compat != NULL && strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0) 457 return (ZPOOL_STATUS_OK); 458 else 459 return (ZPOOL_STATUS_VERSION_OLDER); 460 } 461 462 /* 463 * Usable pool with disabled or superfluous features 464 * (superfluous = beyond what's requested by 'compatibility') 465 */ 466 if (version >= SPA_VERSION_FEATURES) { 467 int i; 468 nvlist_t *feat; 469 470 if (isimport) { 471 feat = fnvlist_lookup_nvlist(config, 472 ZPOOL_CONFIG_LOAD_INFO); 473 if (nvlist_exists(feat, ZPOOL_CONFIG_ENABLED_FEAT)) 474 feat = fnvlist_lookup_nvlist(feat, 475 ZPOOL_CONFIG_ENABLED_FEAT); 476 } else { 477 feat = fnvlist_lookup_nvlist(config, 478 ZPOOL_CONFIG_FEATURE_STATS); 479 } 480 481 /* check against all features, or limited set? */ 482 boolean_t c_features[SPA_FEATURES]; 483 484 switch (zpool_load_compat(compat, c_features, NULL, 0)) { 485 case ZPOOL_COMPATIBILITY_OK: 486 case ZPOOL_COMPATIBILITY_WARNTOKEN: 487 break; 488 default: 489 return (ZPOOL_STATUS_COMPATIBILITY_ERR); 490 } 491 for (i = 0; i < SPA_FEATURES; i++) { 492 zfeature_info_t *fi = &spa_feature_table[i]; 493 if (!fi->fi_zfs_mod_supported || 494 (fi->fi_flags & ZFEATURE_FLAG_NO_UPGRADE)) 495 continue; 496 if (c_features[i] && !nvlist_exists(feat, fi->fi_guid)) 497 return (ZPOOL_STATUS_FEAT_DISABLED); 498 if (!c_features[i] && nvlist_exists(feat, fi->fi_guid)) 499 return (ZPOOL_STATUS_INCOMPATIBLE_FEAT); 500 } 501 } 502 503 return (ZPOOL_STATUS_OK); 504 } 505 506 zpool_status_t 507 zpool_get_status(zpool_handle_t *zhp, const char **msgid, 508 zpool_errata_t *errata) 509 { 510 /* 511 * pass in the desired feature set, as 512 * it affects check for disabled features 513 */ 514 char compatibility[ZFS_MAXPROPLEN]; 515 if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compatibility, 516 ZFS_MAXPROPLEN, NULL, B_FALSE) != 0) 517 compatibility[0] = '\0'; 518 519 uint64_t ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, NULL); 520 521 zpool_status_t ret = check_status(zhp->zpool_config, B_FALSE, errata, 522 compatibility, ashift); 523 524 if (msgid != NULL) { 525 if (ret >= NMSGID) 526 *msgid = NULL; 527 else 528 *msgid = zfs_msgid_table[ret]; 529 } 530 return (ret); 531 } 532 533 zpool_status_t 534 zpool_import_status(nvlist_t *config, const char **msgid, 535 zpool_errata_t *errata) 536 { 537 zpool_status_t ret = check_status(config, B_TRUE, errata, NULL, 0); 538 539 if (ret >= NMSGID) 540 *msgid = NULL; 541 else 542 *msgid = zfs_msgid_table[ret]; 543 544 return (ret); 545 } 546