// SPDX-License-Identifier: GPL-2.0 #include "bcachefs.h" #include "btree_iter.h" #include "error.h" #include "journal.h" #include "recovery_passes.h" #include "super.h" #include "thread_with_file.h" #define FSCK_ERR_RATELIMIT_NR 10 bool bch2_inconsistent_error(struct bch_fs *c) { set_bit(BCH_FS_error, &c->flags); switch (c->opts.errors) { case BCH_ON_ERROR_continue: return false; case BCH_ON_ERROR_fix_safe: case BCH_ON_ERROR_ro: if (bch2_fs_emergency_read_only(c)) bch_err(c, "inconsistency detected - emergency read only at journal seq %llu", journal_cur_seq(&c->journal)); return true; case BCH_ON_ERROR_panic: panic(bch2_fmt(c, "panic after error")); return true; default: BUG(); } } int bch2_topology_error(struct bch_fs *c) { set_bit(BCH_FS_topology_error, &c->flags); if (!test_bit(BCH_FS_fsck_running, &c->flags)) { bch2_inconsistent_error(c); return -BCH_ERR_btree_need_topology_repair; } else { return bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology) ?: -BCH_ERR_btree_node_read_validate_error; } } void bch2_fatal_error(struct bch_fs *c) { if (bch2_fs_emergency_read_only(c)) bch_err(c, "fatal error - emergency read only"); } void bch2_io_error_work(struct work_struct *work) { struct bch_dev *ca = container_of(work, struct bch_dev, io_error_work); struct bch_fs *c = ca->fs; bool dev; down_write(&c->state_lock); dev = bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_ro, BCH_FORCE_IF_DEGRADED); if (dev ? __bch2_dev_set_state(c, ca, BCH_MEMBER_STATE_ro, BCH_FORCE_IF_DEGRADED) : bch2_fs_emergency_read_only(c)) bch_err(ca, "too many IO errors, setting %s RO", dev ? "device" : "filesystem"); up_write(&c->state_lock); } void bch2_io_error(struct bch_dev *ca, enum bch_member_error_type type) { atomic64_inc(&ca->errors[type]); //queue_work(system_long_wq, &ca->io_error_work); } enum ask_yn { YN_NO, YN_YES, YN_ALLNO, YN_ALLYES, }; static enum ask_yn parse_yn_response(char *buf) { buf = strim(buf); if (strlen(buf) == 1) switch (buf[0]) { case 'n': return YN_NO; case 'y': return YN_YES; case 'N': return YN_ALLNO; case 'Y': return YN_ALLYES; } return -1; } #ifdef __KERNEL__ static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans) { struct stdio_redirect *stdio = c->stdio; if (c->stdio_filter && c->stdio_filter != current) stdio = NULL; if (!stdio) return YN_NO; if (trans) bch2_trans_unlock(trans); unsigned long unlock_long_at = trans ? jiffies + HZ * 2 : 0; darray_char line = {}; int ret; do { unsigned long t; bch2_print(c, " (y,n, or Y,N for all errors of this type) "); rewait: t = unlock_long_at ? max_t(long, unlock_long_at - jiffies, 0) : MAX_SCHEDULE_TIMEOUT; int r = bch2_stdio_redirect_readline_timeout(stdio, &line, t); if (r == -ETIME) { bch2_trans_unlock_long(trans); unlock_long_at = 0; goto rewait; } if (r < 0) { ret = YN_NO; break; } darray_last(line) = '\0'; } while ((ret = parse_yn_response(line.data)) < 0); darray_exit(&line); return ret; } #else #include "tools-util.h" static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans) { char *buf = NULL; size_t buflen = 0; int ret; do { fputs(" (y,n, or Y,N for all errors of this type) ", stdout); fflush(stdout); if (getline(&buf, &buflen, stdin) < 0) die("error reading from standard input"); } while ((ret = parse_yn_response(buf)) < 0); free(buf); return ret; } #endif static struct fsck_err_state *fsck_err_get(struct bch_fs *c, const char *fmt) { struct fsck_err_state *s; if (!test_bit(BCH_FS_fsck_running, &c->flags)) return NULL; list_for_each_entry(s, &c->fsck_error_msgs, list) if (s->fmt == fmt) { /* * move it to the head of the list: repeated fsck errors * are common */ list_move(&s->list, &c->fsck_error_msgs); return s; } s = kzalloc(sizeof(*s), GFP_NOFS); if (!s) { if (!c->fsck_alloc_msgs_err) bch_err(c, "kmalloc err, cannot ratelimit fsck errs"); c->fsck_alloc_msgs_err = true; return NULL; } INIT_LIST_HEAD(&s->list); s->fmt = fmt; list_add(&s->list, &c->fsck_error_msgs); return s; } /* s/fix?/fixing/ s/recreate?/recreating/ */ static void prt_actioning(struct printbuf *out, const char *action) { unsigned len = strlen(action); BUG_ON(action[len - 1] != '?'); --len; if (action[len - 1] == 'e') --len; prt_bytes(out, action, len); prt_str(out, "ing"); } static const u8 fsck_flags_extra[] = { #define x(t, n, flags) [BCH_FSCK_ERR_##t] = flags, BCH_SB_ERRS() #undef x }; int __bch2_fsck_err(struct bch_fs *c, struct btree_trans *trans, enum bch_fsck_flags flags, enum bch_sb_error_id err, const char *fmt, ...) { struct fsck_err_state *s = NULL; va_list args; bool print = true, suppressing = false, inconsistent = false; struct printbuf buf = PRINTBUF, *out = &buf; int ret = -BCH_ERR_fsck_ignore; const char *action_orig = "fix?", *action = action_orig; might_sleep(); if (!WARN_ON(err >= ARRAY_SIZE(fsck_flags_extra))) flags |= fsck_flags_extra[err]; if (!c) c = trans->c; /* * Ugly: if there's a transaction in the current task it has to be * passed in to unlock if we prompt for user input. * * But, plumbing a transaction and transaction restarts into * bkey_validate() is problematic. * * So: * - make all bkey errors AUTOFIX, they're simple anyways (we just * delete the key) * - and we don't need to warn if we're not prompting */ WARN_ON((flags & FSCK_CAN_FIX) && !(flags & FSCK_AUTOFIX) && !trans && bch2_current_has_btree_trans(c)); if ((flags & FSCK_CAN_FIX) && test_bit(err, c->sb.errors_silent)) return -BCH_ERR_fsck_fix; bch2_sb_error_count(c, err); va_start(args, fmt); prt_vprintf(out, fmt, args); va_end(args); /* Custom fix/continue/recreate/etc.? */ if (out->buf[out->pos - 1] == '?') { const char *p = strrchr(out->buf, ','); if (p) { out->pos = p - out->buf; action = kstrdup(p + 2, GFP_KERNEL); if (!action) { ret = -ENOMEM; goto err; } } } mutex_lock(&c->fsck_error_msgs_lock); s = fsck_err_get(c, fmt); if (s) { /* * We may be called multiple times for the same error on * transaction restart - this memoizes instead of asking the user * multiple times for the same error: */ if (s->last_msg && !strcmp(buf.buf, s->last_msg)) { ret = s->ret; mutex_unlock(&c->fsck_error_msgs_lock); goto err; } kfree(s->last_msg); s->last_msg = kstrdup(buf.buf, GFP_KERNEL); if (!s->last_msg) { mutex_unlock(&c->fsck_error_msgs_lock); ret = -ENOMEM; goto err; } if (c->opts.ratelimit_errors && !(flags & FSCK_NO_RATELIMIT) && s->nr >= FSCK_ERR_RATELIMIT_NR) { if (s->nr == FSCK_ERR_RATELIMIT_NR) suppressing = true; else print = false; } s->nr++; } #ifdef BCACHEFS_LOG_PREFIX if (!strncmp(fmt, "bcachefs:", 9)) prt_printf(out, bch2_log_msg(c, "")); #endif if ((flags & FSCK_CAN_FIX) && (flags & FSCK_AUTOFIX) && (c->opts.errors == BCH_ON_ERROR_continue || c->opts.errors == BCH_ON_ERROR_fix_safe)) { prt_str(out, ", "); prt_actioning(out, action); ret = -BCH_ERR_fsck_fix; } else if (!test_bit(BCH_FS_fsck_running, &c->flags)) { if (c->opts.errors != BCH_ON_ERROR_continue || !(flags & (FSCK_CAN_FIX|FSCK_CAN_IGNORE))) { prt_str(out, ", shutting down"); inconsistent = true; ret = -BCH_ERR_fsck_errors_not_fixed; } else if (flags & FSCK_CAN_FIX) { prt_str(out, ", "); prt_actioning(out, action); ret = -BCH_ERR_fsck_fix; } else { prt_str(out, ", continuing"); ret = -BCH_ERR_fsck_ignore; } } else if (c->opts.fix_errors == FSCK_FIX_exit) { prt_str(out, ", exiting"); ret = -BCH_ERR_fsck_errors_not_fixed; } else if (flags & FSCK_CAN_FIX) { int fix = s && s->fix ? s->fix : c->opts.fix_errors; if (fix == FSCK_FIX_ask) { prt_str(out, ", "); prt_str(out, action); if (bch2_fs_stdio_redirect(c)) bch2_print(c, "%s", out->buf); else bch2_print_string_as_lines(KERN_ERR, out->buf); print = false; int ask = bch2_fsck_ask_yn(c, trans); if (trans) { ret = bch2_trans_relock(trans); if (ret) { mutex_unlock(&c->fsck_error_msgs_lock); goto err; } } if (ask >= YN_ALLNO && s) s->fix = ask == YN_ALLNO ? FSCK_FIX_no : FSCK_FIX_yes; ret = ask & 1 ? -BCH_ERR_fsck_fix : -BCH_ERR_fsck_ignore; } else if (fix == FSCK_FIX_yes || (c->opts.nochanges && !(flags & FSCK_CAN_IGNORE))) { prt_str(out, ", "); prt_actioning(out, action); ret = -BCH_ERR_fsck_fix; } else { prt_str(out, ", not "); prt_actioning(out, action); } } else if (flags & FSCK_NEED_FSCK) { prt_str(out, " (run fsck to correct)"); } else { prt_str(out, " (repair unimplemented)"); } if (ret == -BCH_ERR_fsck_ignore && (c->opts.fix_errors == FSCK_FIX_exit || !(flags & FSCK_CAN_IGNORE))) ret = -BCH_ERR_fsck_errors_not_fixed; bool exiting = test_bit(BCH_FS_fsck_running, &c->flags) && (ret != -BCH_ERR_fsck_fix && ret != -BCH_ERR_fsck_ignore); if (exiting) print = true; if (print) { if (bch2_fs_stdio_redirect(c)) bch2_print(c, "%s\n", out->buf); else bch2_print_string_as_lines(KERN_ERR, out->buf); } if (exiting) bch_err(c, "Unable to continue, halting"); else if (suppressing) bch_err(c, "Ratelimiting new instances of previous error"); if (s) s->ret = ret; mutex_unlock(&c->fsck_error_msgs_lock); if (inconsistent) bch2_inconsistent_error(c); if (ret == -BCH_ERR_fsck_fix) { set_bit(BCH_FS_errors_fixed, &c->flags); } else { set_bit(BCH_FS_errors_not_fixed, &c->flags); set_bit(BCH_FS_error, &c->flags); } err: if (action != action_orig) kfree(action); printbuf_exit(&buf); return ret; } int __bch2_bkey_fsck_err(struct bch_fs *c, struct bkey_s_c k, enum bch_validate_flags validate_flags, enum bch_sb_error_id err, const char *fmt, ...) { if (validate_flags & BCH_VALIDATE_silent) return -BCH_ERR_fsck_delete_bkey; unsigned fsck_flags = 0; if (!(validate_flags & (BCH_VALIDATE_write|BCH_VALIDATE_commit))) fsck_flags |= FSCK_AUTOFIX|FSCK_CAN_FIX; struct printbuf buf = PRINTBUF; va_list args; prt_str(&buf, "invalid bkey "); bch2_bkey_val_to_text(&buf, c, k); prt_str(&buf, "\n "); va_start(args, fmt); prt_vprintf(&buf, fmt, args); va_end(args); prt_str(&buf, ": delete?"); int ret = __bch2_fsck_err(c, NULL, fsck_flags, err, "%s", buf.buf); printbuf_exit(&buf); return ret; } void bch2_flush_fsck_errs(struct bch_fs *c) { struct fsck_err_state *s, *n; mutex_lock(&c->fsck_error_msgs_lock); list_for_each_entry_safe(s, n, &c->fsck_error_msgs, list) { if (s->ratelimited && s->last_msg) bch_err(c, "Saw %llu errors like:\n %s", s->nr, s->last_msg); list_del(&s->list); kfree(s->last_msg); kfree(s); } mutex_unlock(&c->fsck_error_msgs_lock); }