1 // SPDX-License-Identifier: GPL-2.0
2 #include "bcachefs.h"
3 #include "btree_iter.h"
4 #include "error.h"
5 #include "journal.h"
6 #include "recovery_passes.h"
7 #include "super.h"
8 #include "thread_with_file.h"
9
10 #define FSCK_ERR_RATELIMIT_NR 10
11
bch2_inconsistent_error(struct bch_fs * c)12 bool bch2_inconsistent_error(struct bch_fs *c)
13 {
14 set_bit(BCH_FS_error, &c->flags);
15
16 switch (c->opts.errors) {
17 case BCH_ON_ERROR_continue:
18 return false;
19 case BCH_ON_ERROR_fix_safe:
20 case BCH_ON_ERROR_ro:
21 if (bch2_fs_emergency_read_only(c))
22 bch_err(c, "inconsistency detected - emergency read only at journal seq %llu",
23 journal_cur_seq(&c->journal));
24 return true;
25 case BCH_ON_ERROR_panic:
26 panic(bch2_fmt(c, "panic after error"));
27 return true;
28 default:
29 BUG();
30 }
31 }
32
bch2_topology_error(struct bch_fs * c)33 int bch2_topology_error(struct bch_fs *c)
34 {
35 set_bit(BCH_FS_topology_error, &c->flags);
36 if (!test_bit(BCH_FS_fsck_running, &c->flags)) {
37 bch2_inconsistent_error(c);
38 return -BCH_ERR_btree_need_topology_repair;
39 } else {
40 return bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology) ?:
41 -BCH_ERR_btree_node_read_validate_error;
42 }
43 }
44
bch2_fatal_error(struct bch_fs * c)45 void bch2_fatal_error(struct bch_fs *c)
46 {
47 if (bch2_fs_emergency_read_only(c))
48 bch_err(c, "fatal error - emergency read only");
49 }
50
bch2_io_error_work(struct work_struct * work)51 void bch2_io_error_work(struct work_struct *work)
52 {
53 struct bch_dev *ca = container_of(work, struct bch_dev, io_error_work);
54 struct bch_fs *c = ca->fs;
55 bool dev;
56
57 down_write(&c->state_lock);
58 dev = bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_ro,
59 BCH_FORCE_IF_DEGRADED);
60 if (dev
61 ? __bch2_dev_set_state(c, ca, BCH_MEMBER_STATE_ro,
62 BCH_FORCE_IF_DEGRADED)
63 : bch2_fs_emergency_read_only(c))
64 bch_err(ca,
65 "too many IO errors, setting %s RO",
66 dev ? "device" : "filesystem");
67 up_write(&c->state_lock);
68 }
69
bch2_io_error(struct bch_dev * ca,enum bch_member_error_type type)70 void bch2_io_error(struct bch_dev *ca, enum bch_member_error_type type)
71 {
72 atomic64_inc(&ca->errors[type]);
73 //queue_work(system_long_wq, &ca->io_error_work);
74 }
75
76 enum ask_yn {
77 YN_NO,
78 YN_YES,
79 YN_ALLNO,
80 YN_ALLYES,
81 };
82
parse_yn_response(char * buf)83 static enum ask_yn parse_yn_response(char *buf)
84 {
85 buf = strim(buf);
86
87 if (strlen(buf) == 1)
88 switch (buf[0]) {
89 case 'n':
90 return YN_NO;
91 case 'y':
92 return YN_YES;
93 case 'N':
94 return YN_ALLNO;
95 case 'Y':
96 return YN_ALLYES;
97 }
98 return -1;
99 }
100
101 #ifdef __KERNEL__
bch2_fsck_ask_yn(struct bch_fs * c,struct btree_trans * trans)102 static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans)
103 {
104 struct stdio_redirect *stdio = c->stdio;
105
106 if (c->stdio_filter && c->stdio_filter != current)
107 stdio = NULL;
108
109 if (!stdio)
110 return YN_NO;
111
112 if (trans)
113 bch2_trans_unlock(trans);
114
115 unsigned long unlock_long_at = trans ? jiffies + HZ * 2 : 0;
116 darray_char line = {};
117 int ret;
118
119 do {
120 unsigned long t;
121 bch2_print(c, " (y,n, or Y,N for all errors of this type) ");
122 rewait:
123 t = unlock_long_at
124 ? max_t(long, unlock_long_at - jiffies, 0)
125 : MAX_SCHEDULE_TIMEOUT;
126
127 int r = bch2_stdio_redirect_readline_timeout(stdio, &line, t);
128 if (r == -ETIME) {
129 bch2_trans_unlock_long(trans);
130 unlock_long_at = 0;
131 goto rewait;
132 }
133
134 if (r < 0) {
135 ret = YN_NO;
136 break;
137 }
138
139 darray_last(line) = '\0';
140 } while ((ret = parse_yn_response(line.data)) < 0);
141
142 darray_exit(&line);
143 return ret;
144 }
145 #else
146
147 #include "tools-util.h"
148
bch2_fsck_ask_yn(struct bch_fs * c,struct btree_trans * trans)149 static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans)
150 {
151 char *buf = NULL;
152 size_t buflen = 0;
153 int ret;
154
155 do {
156 fputs(" (y,n, or Y,N for all errors of this type) ", stdout);
157 fflush(stdout);
158
159 if (getline(&buf, &buflen, stdin) < 0)
160 die("error reading from standard input");
161 } while ((ret = parse_yn_response(buf)) < 0);
162
163 free(buf);
164 return ret;
165 }
166
167 #endif
168
fsck_err_get(struct bch_fs * c,const char * fmt)169 static struct fsck_err_state *fsck_err_get(struct bch_fs *c, const char *fmt)
170 {
171 struct fsck_err_state *s;
172
173 if (!test_bit(BCH_FS_fsck_running, &c->flags))
174 return NULL;
175
176 list_for_each_entry(s, &c->fsck_error_msgs, list)
177 if (s->fmt == fmt) {
178 /*
179 * move it to the head of the list: repeated fsck errors
180 * are common
181 */
182 list_move(&s->list, &c->fsck_error_msgs);
183 return s;
184 }
185
186 s = kzalloc(sizeof(*s), GFP_NOFS);
187 if (!s) {
188 if (!c->fsck_alloc_msgs_err)
189 bch_err(c, "kmalloc err, cannot ratelimit fsck errs");
190 c->fsck_alloc_msgs_err = true;
191 return NULL;
192 }
193
194 INIT_LIST_HEAD(&s->list);
195 s->fmt = fmt;
196 list_add(&s->list, &c->fsck_error_msgs);
197 return s;
198 }
199
200 /* s/fix?/fixing/ s/recreate?/recreating/ */
prt_actioning(struct printbuf * out,const char * action)201 static void prt_actioning(struct printbuf *out, const char *action)
202 {
203 unsigned len = strlen(action);
204
205 BUG_ON(action[len - 1] != '?');
206 --len;
207
208 if (action[len - 1] == 'e')
209 --len;
210
211 prt_bytes(out, action, len);
212 prt_str(out, "ing");
213 }
214
215 static const u8 fsck_flags_extra[] = {
216 #define x(t, n, flags) [BCH_FSCK_ERR_##t] = flags,
217 BCH_SB_ERRS()
218 #undef x
219 };
220
__bch2_fsck_err(struct bch_fs * c,struct btree_trans * trans,enum bch_fsck_flags flags,enum bch_sb_error_id err,const char * fmt,...)221 int __bch2_fsck_err(struct bch_fs *c,
222 struct btree_trans *trans,
223 enum bch_fsck_flags flags,
224 enum bch_sb_error_id err,
225 const char *fmt, ...)
226 {
227 struct fsck_err_state *s = NULL;
228 va_list args;
229 bool print = true, suppressing = false, inconsistent = false;
230 struct printbuf buf = PRINTBUF, *out = &buf;
231 int ret = -BCH_ERR_fsck_ignore;
232 const char *action_orig = "fix?", *action = action_orig;
233
234 might_sleep();
235
236 if (!WARN_ON(err >= ARRAY_SIZE(fsck_flags_extra)))
237 flags |= fsck_flags_extra[err];
238
239 if (!c)
240 c = trans->c;
241
242 /*
243 * Ugly: if there's a transaction in the current task it has to be
244 * passed in to unlock if we prompt for user input.
245 *
246 * But, plumbing a transaction and transaction restarts into
247 * bkey_validate() is problematic.
248 *
249 * So:
250 * - make all bkey errors AUTOFIX, they're simple anyways (we just
251 * delete the key)
252 * - and we don't need to warn if we're not prompting
253 */
254 WARN_ON((flags & FSCK_CAN_FIX) &&
255 !(flags & FSCK_AUTOFIX) &&
256 !trans &&
257 bch2_current_has_btree_trans(c));
258
259 if ((flags & FSCK_CAN_FIX) &&
260 test_bit(err, c->sb.errors_silent))
261 return -BCH_ERR_fsck_fix;
262
263 bch2_sb_error_count(c, err);
264
265 va_start(args, fmt);
266 prt_vprintf(out, fmt, args);
267 va_end(args);
268
269 /* Custom fix/continue/recreate/etc.? */
270 if (out->buf[out->pos - 1] == '?') {
271 const char *p = strrchr(out->buf, ',');
272 if (p) {
273 out->pos = p - out->buf;
274 action = kstrdup(p + 2, GFP_KERNEL);
275 if (!action) {
276 ret = -ENOMEM;
277 goto err;
278 }
279 }
280 }
281
282 mutex_lock(&c->fsck_error_msgs_lock);
283 s = fsck_err_get(c, fmt);
284 if (s) {
285 /*
286 * We may be called multiple times for the same error on
287 * transaction restart - this memoizes instead of asking the user
288 * multiple times for the same error:
289 */
290 if (s->last_msg && !strcmp(buf.buf, s->last_msg)) {
291 ret = s->ret;
292 mutex_unlock(&c->fsck_error_msgs_lock);
293 goto err;
294 }
295
296 kfree(s->last_msg);
297 s->last_msg = kstrdup(buf.buf, GFP_KERNEL);
298 if (!s->last_msg) {
299 mutex_unlock(&c->fsck_error_msgs_lock);
300 ret = -ENOMEM;
301 goto err;
302 }
303
304 if (c->opts.ratelimit_errors &&
305 !(flags & FSCK_NO_RATELIMIT) &&
306 s->nr >= FSCK_ERR_RATELIMIT_NR) {
307 if (s->nr == FSCK_ERR_RATELIMIT_NR)
308 suppressing = true;
309 else
310 print = false;
311 }
312
313 s->nr++;
314 }
315
316 #ifdef BCACHEFS_LOG_PREFIX
317 if (!strncmp(fmt, "bcachefs:", 9))
318 prt_printf(out, bch2_log_msg(c, ""));
319 #endif
320
321 if ((flags & FSCK_CAN_FIX) &&
322 (flags & FSCK_AUTOFIX) &&
323 (c->opts.errors == BCH_ON_ERROR_continue ||
324 c->opts.errors == BCH_ON_ERROR_fix_safe)) {
325 prt_str(out, ", ");
326 prt_actioning(out, action);
327 ret = -BCH_ERR_fsck_fix;
328 } else if (!test_bit(BCH_FS_fsck_running, &c->flags)) {
329 if (c->opts.errors != BCH_ON_ERROR_continue ||
330 !(flags & (FSCK_CAN_FIX|FSCK_CAN_IGNORE))) {
331 prt_str(out, ", shutting down");
332 inconsistent = true;
333 ret = -BCH_ERR_fsck_errors_not_fixed;
334 } else if (flags & FSCK_CAN_FIX) {
335 prt_str(out, ", ");
336 prt_actioning(out, action);
337 ret = -BCH_ERR_fsck_fix;
338 } else {
339 prt_str(out, ", continuing");
340 ret = -BCH_ERR_fsck_ignore;
341 }
342 } else if (c->opts.fix_errors == FSCK_FIX_exit) {
343 prt_str(out, ", exiting");
344 ret = -BCH_ERR_fsck_errors_not_fixed;
345 } else if (flags & FSCK_CAN_FIX) {
346 int fix = s && s->fix
347 ? s->fix
348 : c->opts.fix_errors;
349
350 if (fix == FSCK_FIX_ask) {
351 prt_str(out, ", ");
352 prt_str(out, action);
353
354 if (bch2_fs_stdio_redirect(c))
355 bch2_print(c, "%s", out->buf);
356 else
357 bch2_print_string_as_lines(KERN_ERR, out->buf);
358 print = false;
359
360 int ask = bch2_fsck_ask_yn(c, trans);
361
362 if (trans) {
363 ret = bch2_trans_relock(trans);
364 if (ret) {
365 mutex_unlock(&c->fsck_error_msgs_lock);
366 goto err;
367 }
368 }
369
370 if (ask >= YN_ALLNO && s)
371 s->fix = ask == YN_ALLNO
372 ? FSCK_FIX_no
373 : FSCK_FIX_yes;
374
375 ret = ask & 1
376 ? -BCH_ERR_fsck_fix
377 : -BCH_ERR_fsck_ignore;
378 } else if (fix == FSCK_FIX_yes ||
379 (c->opts.nochanges &&
380 !(flags & FSCK_CAN_IGNORE))) {
381 prt_str(out, ", ");
382 prt_actioning(out, action);
383 ret = -BCH_ERR_fsck_fix;
384 } else {
385 prt_str(out, ", not ");
386 prt_actioning(out, action);
387 }
388 } else if (flags & FSCK_NEED_FSCK) {
389 prt_str(out, " (run fsck to correct)");
390 } else {
391 prt_str(out, " (repair unimplemented)");
392 }
393
394 if (ret == -BCH_ERR_fsck_ignore &&
395 (c->opts.fix_errors == FSCK_FIX_exit ||
396 !(flags & FSCK_CAN_IGNORE)))
397 ret = -BCH_ERR_fsck_errors_not_fixed;
398
399 bool exiting =
400 test_bit(BCH_FS_fsck_running, &c->flags) &&
401 (ret != -BCH_ERR_fsck_fix &&
402 ret != -BCH_ERR_fsck_ignore);
403
404 if (exiting)
405 print = true;
406
407 if (print) {
408 if (bch2_fs_stdio_redirect(c))
409 bch2_print(c, "%s\n", out->buf);
410 else
411 bch2_print_string_as_lines(KERN_ERR, out->buf);
412 }
413
414 if (exiting)
415 bch_err(c, "Unable to continue, halting");
416 else if (suppressing)
417 bch_err(c, "Ratelimiting new instances of previous error");
418
419 if (s)
420 s->ret = ret;
421
422 mutex_unlock(&c->fsck_error_msgs_lock);
423
424 if (inconsistent)
425 bch2_inconsistent_error(c);
426
427 if (ret == -BCH_ERR_fsck_fix) {
428 set_bit(BCH_FS_errors_fixed, &c->flags);
429 } else {
430 set_bit(BCH_FS_errors_not_fixed, &c->flags);
431 set_bit(BCH_FS_error, &c->flags);
432 }
433 err:
434 if (action != action_orig)
435 kfree(action);
436 printbuf_exit(&buf);
437 return ret;
438 }
439
__bch2_bkey_fsck_err(struct bch_fs * c,struct bkey_s_c k,enum bch_validate_flags validate_flags,enum bch_sb_error_id err,const char * fmt,...)440 int __bch2_bkey_fsck_err(struct bch_fs *c,
441 struct bkey_s_c k,
442 enum bch_validate_flags validate_flags,
443 enum bch_sb_error_id err,
444 const char *fmt, ...)
445 {
446 if (validate_flags & BCH_VALIDATE_silent)
447 return -BCH_ERR_fsck_delete_bkey;
448
449 unsigned fsck_flags = 0;
450 if (!(validate_flags & (BCH_VALIDATE_write|BCH_VALIDATE_commit)))
451 fsck_flags |= FSCK_AUTOFIX|FSCK_CAN_FIX;
452
453 struct printbuf buf = PRINTBUF;
454 va_list args;
455
456 prt_str(&buf, "invalid bkey ");
457 bch2_bkey_val_to_text(&buf, c, k);
458 prt_str(&buf, "\n ");
459 va_start(args, fmt);
460 prt_vprintf(&buf, fmt, args);
461 va_end(args);
462 prt_str(&buf, ": delete?");
463
464 int ret = __bch2_fsck_err(c, NULL, fsck_flags, err, "%s", buf.buf);
465 printbuf_exit(&buf);
466 return ret;
467 }
468
bch2_flush_fsck_errs(struct bch_fs * c)469 void bch2_flush_fsck_errs(struct bch_fs *c)
470 {
471 struct fsck_err_state *s, *n;
472
473 mutex_lock(&c->fsck_error_msgs_lock);
474
475 list_for_each_entry_safe(s, n, &c->fsck_error_msgs, list) {
476 if (s->ratelimited && s->last_msg)
477 bch_err(c, "Saw %llu errors like:\n %s", s->nr, s->last_msg);
478
479 list_del(&s->list);
480 kfree(s->last_msg);
481 kfree(s);
482 }
483
484 mutex_unlock(&c->fsck_error_msgs_lock);
485 }
486