11c6fdbd8SKent Overstreet // SPDX-License-Identifier: GPL-2.0 21c6fdbd8SKent Overstreet #include "bcachefs.h" 31c6fdbd8SKent Overstreet #include "error.h" 41c6fdbd8SKent Overstreet #include "super.h" 51c6fdbd8SKent Overstreet 689b05118SKent Overstreet #define FSCK_ERR_RATELIMIT_NR 10 789b05118SKent Overstreet 81c6fdbd8SKent Overstreet bool bch2_inconsistent_error(struct bch_fs *c) 91c6fdbd8SKent Overstreet { 101c6fdbd8SKent Overstreet set_bit(BCH_FS_ERROR, &c->flags); 111c6fdbd8SKent Overstreet 121c6fdbd8SKent Overstreet switch (c->opts.errors) { 132436cb9fSKent Overstreet case BCH_ON_ERROR_continue: 141c6fdbd8SKent Overstreet return false; 152436cb9fSKent Overstreet case BCH_ON_ERROR_ro: 161c6fdbd8SKent Overstreet if (bch2_fs_emergency_read_only(c)) 17b74b147dSKent Overstreet bch_err(c, "inconsistency detected - emergency read only"); 181c6fdbd8SKent Overstreet return true; 192436cb9fSKent Overstreet case BCH_ON_ERROR_panic: 201c6fdbd8SKent Overstreet panic(bch2_fmt(c, "panic after error")); 211c6fdbd8SKent Overstreet return true; 221c6fdbd8SKent Overstreet default: 231c6fdbd8SKent Overstreet BUG(); 241c6fdbd8SKent Overstreet } 251c6fdbd8SKent Overstreet } 261c6fdbd8SKent Overstreet 27aae15aafSKent Overstreet void bch2_topology_error(struct bch_fs *c) 28aae15aafSKent Overstreet { 29aae15aafSKent Overstreet set_bit(BCH_FS_TOPOLOGY_ERROR, &c->flags); 302e984040SKent Overstreet if (test_bit(BCH_FS_FSCK_DONE, &c->flags)) 31aae15aafSKent Overstreet bch2_inconsistent_error(c); 32aae15aafSKent Overstreet } 33aae15aafSKent Overstreet 341c6fdbd8SKent Overstreet void bch2_fatal_error(struct bch_fs *c) 351c6fdbd8SKent Overstreet { 361c6fdbd8SKent Overstreet if (bch2_fs_emergency_read_only(c)) 37b74b147dSKent Overstreet bch_err(c, "fatal error - emergency read only"); 381c6fdbd8SKent Overstreet } 391c6fdbd8SKent Overstreet 401c6fdbd8SKent Overstreet void bch2_io_error_work(struct work_struct *work) 411c6fdbd8SKent Overstreet { 421c6fdbd8SKent Overstreet struct bch_dev *ca = container_of(work, struct bch_dev, io_error_work); 431c6fdbd8SKent Overstreet struct bch_fs *c = ca->fs; 441c6fdbd8SKent Overstreet bool dev; 451c6fdbd8SKent Overstreet 461ada1606SKent Overstreet down_write(&c->state_lock); 472436cb9fSKent Overstreet dev = bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_ro, 481c6fdbd8SKent Overstreet BCH_FORCE_IF_DEGRADED); 491c6fdbd8SKent Overstreet if (dev 502436cb9fSKent Overstreet ? __bch2_dev_set_state(c, ca, BCH_MEMBER_STATE_ro, 511c6fdbd8SKent Overstreet BCH_FORCE_IF_DEGRADED) 521c6fdbd8SKent Overstreet : bch2_fs_emergency_read_only(c)) 531c6fdbd8SKent Overstreet bch_err(ca, 541c6fdbd8SKent Overstreet "too many IO errors, setting %s RO", 551c6fdbd8SKent Overstreet dev ? "device" : "filesystem"); 561ada1606SKent Overstreet up_write(&c->state_lock); 571c6fdbd8SKent Overstreet } 581c6fdbd8SKent Overstreet 5994119eebSKent Overstreet void bch2_io_error(struct bch_dev *ca, enum bch_member_error_type type) 601c6fdbd8SKent Overstreet { 6194119eebSKent Overstreet atomic64_inc(&ca->errors[type]); 621c6fdbd8SKent Overstreet //queue_work(system_long_wq, &ca->io_error_work); 631c6fdbd8SKent Overstreet } 641c6fdbd8SKent Overstreet 65853b7393SKent Overstreet enum ask_yn { 66853b7393SKent Overstreet YN_NO, 67853b7393SKent Overstreet YN_YES, 68853b7393SKent Overstreet YN_ALLNO, 69853b7393SKent Overstreet YN_ALLYES, 70853b7393SKent Overstreet }; 71853b7393SKent Overstreet 721c6fdbd8SKent Overstreet #ifdef __KERNEL__ 73853b7393SKent Overstreet #define bch2_fsck_ask_yn() YN_NO 741c6fdbd8SKent Overstreet #else 75853b7393SKent Overstreet 761c6fdbd8SKent Overstreet #include "tools-util.h" 77853b7393SKent Overstreet 78853b7393SKent Overstreet enum ask_yn bch2_fsck_ask_yn(void) 79853b7393SKent Overstreet { 80853b7393SKent Overstreet char *buf = NULL; 81853b7393SKent Overstreet size_t buflen = 0; 82853b7393SKent Overstreet bool ret; 83853b7393SKent Overstreet 84853b7393SKent Overstreet while (true) { 854f2c166eSKent Overstreet fputs(" (y,n, or Y,N for all errors of this type) ", stdout); 86853b7393SKent Overstreet fflush(stdout); 87853b7393SKent Overstreet 88853b7393SKent Overstreet if (getline(&buf, &buflen, stdin) < 0) 89853b7393SKent Overstreet die("error reading from standard input"); 90853b7393SKent Overstreet 914f2c166eSKent Overstreet strim(buf); 92853b7393SKent Overstreet if (strlen(buf) != 1) 93853b7393SKent Overstreet continue; 94853b7393SKent Overstreet 95853b7393SKent Overstreet switch (buf[0]) { 96853b7393SKent Overstreet case 'n': 97853b7393SKent Overstreet return YN_NO; 98853b7393SKent Overstreet case 'y': 99853b7393SKent Overstreet return YN_YES; 100853b7393SKent Overstreet case 'N': 101853b7393SKent Overstreet return YN_ALLNO; 102853b7393SKent Overstreet case 'Y': 103853b7393SKent Overstreet return YN_ALLYES; 104853b7393SKent Overstreet } 105853b7393SKent Overstreet } 106853b7393SKent Overstreet 107853b7393SKent Overstreet free(buf); 108853b7393SKent Overstreet return ret; 109853b7393SKent Overstreet } 110853b7393SKent Overstreet 1111c6fdbd8SKent Overstreet #endif 1121c6fdbd8SKent Overstreet 113dbb9936bSKent Overstreet static struct fsck_err_state *fsck_err_get(struct bch_fs *c, const char *fmt) 1141c6fdbd8SKent Overstreet { 115dbb9936bSKent Overstreet struct fsck_err_state *s; 1161c6fdbd8SKent Overstreet 117dbb9936bSKent Overstreet if (test_bit(BCH_FS_FSCK_DONE, &c->flags)) 118dbb9936bSKent Overstreet return NULL; 1191c6fdbd8SKent Overstreet 120*f5d26fa3SKent Overstreet list_for_each_entry(s, &c->fsck_error_msgs, list) 121dbb9936bSKent Overstreet if (s->fmt == fmt) { 122dbb9936bSKent Overstreet /* 123dbb9936bSKent Overstreet * move it to the head of the list: repeated fsck errors 124dbb9936bSKent Overstreet * are common 125dbb9936bSKent Overstreet */ 126*f5d26fa3SKent Overstreet list_move(&s->list, &c->fsck_error_msgs); 127dbb9936bSKent Overstreet return s; 128dbb9936bSKent Overstreet } 1291c6fdbd8SKent Overstreet 130beb6db68SKent Overstreet s = kzalloc(sizeof(*s), GFP_NOFS); 1311c6fdbd8SKent Overstreet if (!s) { 132*f5d26fa3SKent Overstreet if (!c->fsck_alloc_msgs_err) 1331c6fdbd8SKent Overstreet bch_err(c, "kmalloc err, cannot ratelimit fsck errs"); 134*f5d26fa3SKent Overstreet c->fsck_alloc_msgs_err = true; 135dbb9936bSKent Overstreet return NULL; 1361c6fdbd8SKent Overstreet } 1371c6fdbd8SKent Overstreet 1381c6fdbd8SKent Overstreet INIT_LIST_HEAD(&s->list); 1391c6fdbd8SKent Overstreet s->fmt = fmt; 140*f5d26fa3SKent Overstreet list_add(&s->list, &c->fsck_error_msgs); 141dbb9936bSKent Overstreet return s; 142dbb9936bSKent Overstreet } 143dbb9936bSKent Overstreet 144dbb9936bSKent Overstreet int bch2_fsck_err(struct bch_fs *c, unsigned flags, const char *fmt, ...) 145dbb9936bSKent Overstreet { 146dbb9936bSKent Overstreet struct fsck_err_state *s = NULL; 147dbb9936bSKent Overstreet va_list args; 14822f51621SKent Overstreet bool print = true, suppressing = false, inconsistent = false; 149dbb9936bSKent Overstreet struct printbuf buf = PRINTBUF, *out = &buf; 150dbb9936bSKent Overstreet int ret = -BCH_ERR_fsck_ignore; 151dbb9936bSKent Overstreet 1529c5d38bbSKent Overstreet va_start(args, fmt); 1539c5d38bbSKent Overstreet prt_vprintf(out, fmt, args); 1549c5d38bbSKent Overstreet va_end(args); 1559c5d38bbSKent Overstreet 156*f5d26fa3SKent Overstreet mutex_lock(&c->fsck_error_msgs_lock); 157dbb9936bSKent Overstreet s = fsck_err_get(c, fmt); 158dbb9936bSKent Overstreet if (s) { 159c8d5b714SKent Overstreet /* 160c8d5b714SKent Overstreet * We may be called multiple times for the same error on 161c8d5b714SKent Overstreet * transaction restart - this memoizes instead of asking the user 162c8d5b714SKent Overstreet * multiple times for the same error: 163c8d5b714SKent Overstreet */ 1649c5d38bbSKent Overstreet if (s->last_msg && !strcmp(buf.buf, s->last_msg)) { 1659c5d38bbSKent Overstreet ret = s->ret; 166*f5d26fa3SKent Overstreet mutex_unlock(&c->fsck_error_msgs_lock); 1679c5d38bbSKent Overstreet printbuf_exit(&buf); 1689c5d38bbSKent Overstreet return ret; 1699c5d38bbSKent Overstreet } 1709c5d38bbSKent Overstreet 1719c5d38bbSKent Overstreet kfree(s->last_msg); 1729c5d38bbSKent Overstreet s->last_msg = kstrdup(buf.buf, GFP_KERNEL); 1739c5d38bbSKent Overstreet 174e2ee3eaaSKent Overstreet if (c->opts.ratelimit_errors && 17508061519SKent Overstreet !(flags & FSCK_NO_RATELIMIT) && 176e2ee3eaaSKent Overstreet s->nr >= FSCK_ERR_RATELIMIT_NR) { 177e2ee3eaaSKent Overstreet if (s->nr == FSCK_ERR_RATELIMIT_NR) 178e2ee3eaaSKent Overstreet suppressing = true; 179e2ee3eaaSKent Overstreet else 180e2ee3eaaSKent Overstreet print = false; 181e2ee3eaaSKent Overstreet } 182dbb9936bSKent Overstreet 183dbb9936bSKent Overstreet s->nr++; 184dbb9936bSKent Overstreet } 185dbb9936bSKent Overstreet 186b2d1d56bSKent Overstreet #ifdef BCACHEFS_LOG_PREFIX 187dbb9936bSKent Overstreet if (!strncmp(fmt, "bcachefs:", 9)) 188dbb9936bSKent Overstreet prt_printf(out, bch2_log_msg(c, "")); 189b2d1d56bSKent Overstreet #endif 190dbb9936bSKent Overstreet 191dbb9936bSKent Overstreet if (test_bit(BCH_FS_FSCK_DONE, &c->flags)) { 192dbb9936bSKent Overstreet if (c->opts.errors != BCH_ON_ERROR_continue || 193dbb9936bSKent Overstreet !(flags & (FSCK_CAN_FIX|FSCK_CAN_IGNORE))) { 194dbb9936bSKent Overstreet prt_str(out, ", shutting down"); 19522f51621SKent Overstreet inconsistent = true; 196dbb9936bSKent Overstreet ret = -BCH_ERR_fsck_errors_not_fixed; 197dbb9936bSKent Overstreet } else if (flags & FSCK_CAN_FIX) { 198dbb9936bSKent Overstreet prt_str(out, ", fixing"); 199dbb9936bSKent Overstreet ret = -BCH_ERR_fsck_fix; 200dbb9936bSKent Overstreet } else { 201dbb9936bSKent Overstreet prt_str(out, ", continuing"); 202dbb9936bSKent Overstreet ret = -BCH_ERR_fsck_ignore; 203dbb9936bSKent Overstreet } 204a0f8faeaSKent Overstreet } else if (c->opts.fix_errors == FSCK_FIX_exit) { 205dbb9936bSKent Overstreet prt_str(out, ", exiting"); 206dbb9936bSKent Overstreet ret = -BCH_ERR_fsck_errors_not_fixed; 2070bc166ffSKent Overstreet } else if (flags & FSCK_CAN_FIX) { 208853b7393SKent Overstreet int fix = s && s->fix 209853b7393SKent Overstreet ? s->fix 210853b7393SKent Overstreet : c->opts.fix_errors; 211853b7393SKent Overstreet 212a0f8faeaSKent Overstreet if (fix == FSCK_FIX_ask) { 213853b7393SKent Overstreet int ask; 214853b7393SKent Overstreet 215dbb9936bSKent Overstreet prt_str(out, ": fix?"); 216dbb9936bSKent Overstreet bch2_print_string_as_lines(KERN_ERR, out->buf); 217dbb9936bSKent Overstreet print = false; 218853b7393SKent Overstreet 219853b7393SKent Overstreet ask = bch2_fsck_ask_yn(); 220853b7393SKent Overstreet 221853b7393SKent Overstreet if (ask >= YN_ALLNO && s) 222853b7393SKent Overstreet s->fix = ask == YN_ALLNO 223a0f8faeaSKent Overstreet ? FSCK_FIX_no 224a0f8faeaSKent Overstreet : FSCK_FIX_yes; 225853b7393SKent Overstreet 226853b7393SKent Overstreet ret = ask & 1 227dbb9936bSKent Overstreet ? -BCH_ERR_fsck_fix 228dbb9936bSKent Overstreet : -BCH_ERR_fsck_ignore; 229a0f8faeaSKent Overstreet } else if (fix == FSCK_FIX_yes || 2301c6fdbd8SKent Overstreet (c->opts.nochanges && 2311c6fdbd8SKent Overstreet !(flags & FSCK_CAN_IGNORE))) { 232dbb9936bSKent Overstreet prt_str(out, ", fixing"); 233dbb9936bSKent Overstreet ret = -BCH_ERR_fsck_fix; 2341c6fdbd8SKent Overstreet } else { 235dbb9936bSKent Overstreet prt_str(out, ", not fixing"); 2361c6fdbd8SKent Overstreet } 2371c6fdbd8SKent Overstreet } else if (flags & FSCK_NEED_FSCK) { 238dbb9936bSKent Overstreet prt_str(out, " (run fsck to correct)"); 2391c6fdbd8SKent Overstreet } else { 240dbb9936bSKent Overstreet prt_str(out, " (repair unimplemented)"); 2411c6fdbd8SKent Overstreet } 2421c6fdbd8SKent Overstreet 243dbb9936bSKent Overstreet if (ret == -BCH_ERR_fsck_ignore && 244a0f8faeaSKent Overstreet (c->opts.fix_errors == FSCK_FIX_exit || 245dbb9936bSKent Overstreet !(flags & FSCK_CAN_IGNORE))) 246dbb9936bSKent Overstreet ret = -BCH_ERR_fsck_errors_not_fixed; 247dbb9936bSKent Overstreet 248dbb9936bSKent Overstreet if (print) 249dbb9936bSKent Overstreet bch2_print_string_as_lines(KERN_ERR, out->buf); 250dbb9936bSKent Overstreet 251dbb9936bSKent Overstreet if (!test_bit(BCH_FS_FSCK_DONE, &c->flags) && 252dbb9936bSKent Overstreet (ret != -BCH_ERR_fsck_fix && 253dbb9936bSKent Overstreet ret != -BCH_ERR_fsck_ignore)) 254dbb9936bSKent Overstreet bch_err(c, "Unable to continue, halting"); 255dbb9936bSKent Overstreet else if (suppressing) 2561c6fdbd8SKent Overstreet bch_err(c, "Ratelimiting new instances of previous error"); 2571c6fdbd8SKent Overstreet 2589c5d38bbSKent Overstreet if (s) 2599c5d38bbSKent Overstreet s->ret = ret; 2609c5d38bbSKent Overstreet 261*f5d26fa3SKent Overstreet mutex_unlock(&c->fsck_error_msgs_lock); 2621c6fdbd8SKent Overstreet 263dbb9936bSKent Overstreet printbuf_exit(&buf); 264dbb9936bSKent Overstreet 26522f51621SKent Overstreet if (inconsistent) 26622f51621SKent Overstreet bch2_inconsistent_error(c); 26722f51621SKent Overstreet 268dbb9936bSKent Overstreet if (ret == -BCH_ERR_fsck_fix) { 2690bc166ffSKent Overstreet set_bit(BCH_FS_ERRORS_FIXED, &c->flags); 2700bc166ffSKent Overstreet } else { 271aae15aafSKent Overstreet set_bit(BCH_FS_ERRORS_NOT_FIXED, &c->flags); 2720bc166ffSKent Overstreet set_bit(BCH_FS_ERROR, &c->flags); 2730bc166ffSKent Overstreet } 274dbb9936bSKent Overstreet 275dbb9936bSKent Overstreet return ret; 2761c6fdbd8SKent Overstreet } 2771c6fdbd8SKent Overstreet 2781c6fdbd8SKent Overstreet void bch2_flush_fsck_errs(struct bch_fs *c) 2791c6fdbd8SKent Overstreet { 2801c6fdbd8SKent Overstreet struct fsck_err_state *s, *n; 2811c6fdbd8SKent Overstreet 282*f5d26fa3SKent Overstreet mutex_lock(&c->fsck_error_msgs_lock); 2831c6fdbd8SKent Overstreet 284*f5d26fa3SKent Overstreet list_for_each_entry_safe(s, n, &c->fsck_error_msgs, list) { 2859c5d38bbSKent Overstreet if (s->ratelimited && s->last_msg) 2869c5d38bbSKent Overstreet bch_err(c, "Saw %llu errors like:\n %s", s->nr, s->last_msg); 2871c6fdbd8SKent Overstreet 2881c6fdbd8SKent Overstreet list_del(&s->list); 2899c5d38bbSKent Overstreet kfree(s->last_msg); 2901c6fdbd8SKent Overstreet kfree(s); 2911c6fdbd8SKent Overstreet } 2921c6fdbd8SKent Overstreet 293*f5d26fa3SKent Overstreet mutex_unlock(&c->fsck_error_msgs_lock); 2941c6fdbd8SKent Overstreet } 295