1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "bcachefs.h" 4 #include "alloc_background.h" 5 #include "backpointers.h" 6 #include "btree_gc.h" 7 #include "btree_node_scan.h" 8 #include "disk_accounting.h" 9 #include "ec.h" 10 #include "fsck.h" 11 #include "inode.h" 12 #include "journal.h" 13 #include "lru.h" 14 #include "logged_ops.h" 15 #include "rebalance.h" 16 #include "recovery.h" 17 #include "recovery_passes.h" 18 #include "snapshot.h" 19 #include "subvolume.h" 20 #include "super.h" 21 #include "super-io.h" 22 23 const char * const bch2_recovery_passes[] = { 24 #define x(_fn, ...) #_fn, 25 BCH_RECOVERY_PASSES() 26 #undef x 27 NULL 28 }; 29 30 /* Fake recovery pass, so that scan_for_btree_nodes isn't 0: */ 31 static int bch2_recovery_pass_empty(struct bch_fs *c) 32 { 33 return 0; 34 } 35 36 static int bch2_set_may_go_rw(struct bch_fs *c) 37 { 38 struct journal_keys *keys = &c->journal_keys; 39 40 /* 41 * After we go RW, the journal keys buffer can't be modified (except for 42 * setting journal_key->overwritten: it will be accessed by multiple 43 * threads 44 */ 45 move_gap(keys, keys->nr); 46 47 set_bit(BCH_FS_may_go_rw, &c->flags); 48 49 if (keys->nr || !c->opts.read_only || c->opts.fsck || !c->sb.clean || c->opts.recovery_passes) 50 return bch2_fs_read_write_early(c); 51 return 0; 52 } 53 54 struct recovery_pass_fn { 55 int (*fn)(struct bch_fs *); 56 unsigned when; 57 }; 58 59 static struct recovery_pass_fn recovery_pass_fns[] = { 60 #define x(_fn, _id, _when) { .fn = bch2_##_fn, .when = _when }, 61 BCH_RECOVERY_PASSES() 62 #undef x 63 }; 64 65 static const u8 passes_to_stable_map[] = { 66 #define x(n, id, ...) [BCH_RECOVERY_PASS_##n] = BCH_RECOVERY_PASS_STABLE_##n, 67 BCH_RECOVERY_PASSES() 68 #undef x 69 }; 70 71 static enum bch_recovery_pass_stable bch2_recovery_pass_to_stable(enum bch_recovery_pass pass) 72 { 73 return passes_to_stable_map[pass]; 74 } 75 76 u64 bch2_recovery_passes_to_stable(u64 v) 77 { 78 u64 ret = 0; 79 for (unsigned i = 0; i < ARRAY_SIZE(passes_to_stable_map); i++) 80 if (v & BIT_ULL(i)) 81 ret |= BIT_ULL(passes_to_stable_map[i]); 82 return ret; 83 } 84 85 u64 bch2_recovery_passes_from_stable(u64 v) 86 { 87 static const u8 map[] = { 88 #define x(n, id, ...) [BCH_RECOVERY_PASS_STABLE_##n] = BCH_RECOVERY_PASS_##n, 89 BCH_RECOVERY_PASSES() 90 #undef x 91 }; 92 93 u64 ret = 0; 94 for (unsigned i = 0; i < ARRAY_SIZE(map); i++) 95 if (v & BIT_ULL(i)) 96 ret |= BIT_ULL(map[i]); 97 return ret; 98 } 99 100 /* 101 * For when we need to rewind recovery passes and run a pass we skipped: 102 */ 103 static int __bch2_run_explicit_recovery_pass(struct bch_fs *c, 104 enum bch_recovery_pass pass) 105 { 106 if (c->curr_recovery_pass == ARRAY_SIZE(recovery_pass_fns)) 107 return -BCH_ERR_not_in_recovery; 108 109 if (c->recovery_passes_complete & BIT_ULL(pass)) 110 return 0; 111 112 bool print = !(c->opts.recovery_passes & BIT_ULL(pass)); 113 114 if (pass < BCH_RECOVERY_PASS_set_may_go_rw && 115 c->curr_recovery_pass >= BCH_RECOVERY_PASS_set_may_go_rw) { 116 if (print) 117 bch_info(c, "need recovery pass %s (%u), but already rw", 118 bch2_recovery_passes[pass], pass); 119 return -BCH_ERR_cannot_rewind_recovery; 120 } 121 122 if (print) 123 bch_info(c, "running explicit recovery pass %s (%u), currently at %s (%u)", 124 bch2_recovery_passes[pass], pass, 125 bch2_recovery_passes[c->curr_recovery_pass], c->curr_recovery_pass); 126 127 c->opts.recovery_passes |= BIT_ULL(pass); 128 129 if (c->curr_recovery_pass > pass) { 130 c->next_recovery_pass = pass; 131 c->recovery_passes_complete &= (1ULL << pass) >> 1; 132 return -BCH_ERR_restart_recovery; 133 } else { 134 return 0; 135 } 136 } 137 138 int bch2_run_explicit_recovery_pass(struct bch_fs *c, 139 enum bch_recovery_pass pass) 140 { 141 unsigned long flags; 142 spin_lock_irqsave(&c->recovery_pass_lock, flags); 143 int ret = __bch2_run_explicit_recovery_pass(c, pass); 144 spin_unlock_irqrestore(&c->recovery_pass_lock, flags); 145 return ret; 146 } 147 148 int bch2_run_explicit_recovery_pass_persistent_locked(struct bch_fs *c, 149 enum bch_recovery_pass pass) 150 { 151 lockdep_assert_held(&c->sb_lock); 152 153 struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext); 154 __set_bit_le64(bch2_recovery_pass_to_stable(pass), ext->recovery_passes_required); 155 156 return bch2_run_explicit_recovery_pass(c, pass); 157 } 158 159 int bch2_run_explicit_recovery_pass_persistent(struct bch_fs *c, 160 enum bch_recovery_pass pass) 161 { 162 enum bch_recovery_pass_stable s = bch2_recovery_pass_to_stable(pass); 163 164 mutex_lock(&c->sb_lock); 165 struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext); 166 167 if (!test_bit_le64(s, ext->recovery_passes_required)) { 168 __set_bit_le64(s, ext->recovery_passes_required); 169 bch2_write_super(c); 170 } 171 mutex_unlock(&c->sb_lock); 172 173 return bch2_run_explicit_recovery_pass(c, pass); 174 } 175 176 static void bch2_clear_recovery_pass_required(struct bch_fs *c, 177 enum bch_recovery_pass pass) 178 { 179 enum bch_recovery_pass_stable s = bch2_recovery_pass_to_stable(pass); 180 181 mutex_lock(&c->sb_lock); 182 struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext); 183 184 if (test_bit_le64(s, ext->recovery_passes_required)) { 185 __clear_bit_le64(s, ext->recovery_passes_required); 186 bch2_write_super(c); 187 } 188 mutex_unlock(&c->sb_lock); 189 } 190 191 u64 bch2_fsck_recovery_passes(void) 192 { 193 u64 ret = 0; 194 195 for (unsigned i = 0; i < ARRAY_SIZE(recovery_pass_fns); i++) 196 if (recovery_pass_fns[i].when & PASS_FSCK) 197 ret |= BIT_ULL(i); 198 return ret; 199 } 200 201 static bool should_run_recovery_pass(struct bch_fs *c, enum bch_recovery_pass pass) 202 { 203 struct recovery_pass_fn *p = recovery_pass_fns + pass; 204 205 if (c->opts.recovery_passes_exclude & BIT_ULL(pass)) 206 return false; 207 if (c->opts.recovery_passes & BIT_ULL(pass)) 208 return true; 209 if ((p->when & PASS_FSCK) && c->opts.fsck) 210 return true; 211 if ((p->when & PASS_UNCLEAN) && !c->sb.clean) 212 return true; 213 if (p->when & PASS_ALWAYS) 214 return true; 215 return false; 216 } 217 218 static int bch2_run_recovery_pass(struct bch_fs *c, enum bch_recovery_pass pass) 219 { 220 struct recovery_pass_fn *p = recovery_pass_fns + pass; 221 int ret; 222 223 if (!(p->when & PASS_SILENT)) 224 bch2_print(c, KERN_INFO bch2_log_msg(c, "%s..."), 225 bch2_recovery_passes[pass]); 226 ret = p->fn(c); 227 if (ret) 228 return ret; 229 if (!(p->when & PASS_SILENT)) 230 bch2_print(c, KERN_CONT " done\n"); 231 232 return 0; 233 } 234 235 int bch2_run_online_recovery_passes(struct bch_fs *c) 236 { 237 for (unsigned i = 0; i < ARRAY_SIZE(recovery_pass_fns); i++) { 238 struct recovery_pass_fn *p = recovery_pass_fns + i; 239 240 if (!(p->when & PASS_ONLINE)) 241 continue; 242 243 int ret = bch2_run_recovery_pass(c, i); 244 if (bch2_err_matches(ret, BCH_ERR_restart_recovery)) { 245 i = c->curr_recovery_pass; 246 continue; 247 } 248 if (ret) 249 return ret; 250 } 251 252 return 0; 253 } 254 255 int bch2_run_recovery_passes(struct bch_fs *c) 256 { 257 int ret = 0; 258 259 /* 260 * We can't allow set_may_go_rw to be excluded; that would cause us to 261 * use the journal replay keys for updates where it's not expected. 262 */ 263 c->opts.recovery_passes_exclude &= ~BCH_RECOVERY_PASS_set_may_go_rw; 264 265 while (c->curr_recovery_pass < ARRAY_SIZE(recovery_pass_fns) && !ret) { 266 c->next_recovery_pass = c->curr_recovery_pass + 1; 267 268 spin_lock_irq(&c->recovery_pass_lock); 269 unsigned pass = c->curr_recovery_pass; 270 271 if (c->opts.recovery_pass_last && 272 c->curr_recovery_pass > c->opts.recovery_pass_last) { 273 spin_unlock_irq(&c->recovery_pass_lock); 274 break; 275 } 276 277 if (!should_run_recovery_pass(c, pass)) { 278 c->curr_recovery_pass++; 279 c->recovery_pass_done = max(c->recovery_pass_done, pass); 280 spin_unlock_irq(&c->recovery_pass_lock); 281 continue; 282 } 283 spin_unlock_irq(&c->recovery_pass_lock); 284 285 ret = bch2_run_recovery_pass(c, pass) ?: 286 bch2_journal_flush(&c->journal); 287 288 if (!ret && !test_bit(BCH_FS_error, &c->flags)) 289 bch2_clear_recovery_pass_required(c, pass); 290 291 spin_lock_irq(&c->recovery_pass_lock); 292 if (c->next_recovery_pass < c->curr_recovery_pass) { 293 /* 294 * bch2_run_explicit_recovery_pass() was called: we 295 * can't always catch -BCH_ERR_restart_recovery because 296 * it may have been called from another thread (btree 297 * node read completion) 298 */ 299 ret = 0; 300 c->recovery_passes_complete &= ~(~0ULL << c->curr_recovery_pass); 301 } else { 302 c->recovery_passes_complete |= BIT_ULL(pass); 303 c->recovery_pass_done = max(c->recovery_pass_done, pass); 304 } 305 c->curr_recovery_pass = c->next_recovery_pass; 306 spin_unlock_irq(&c->recovery_pass_lock); 307 } 308 309 return ret; 310 } 311