1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2010 Kent Overstreet <kent.overstreet@gmail.com> 4 * Copyright (C) 2014 Datera Inc. 5 */ 6 7 #include "bcachefs.h" 8 #include "alloc_background.h" 9 #include "alloc_foreground.h" 10 #include "backpointers.h" 11 #include "bkey_methods.h" 12 #include "bkey_buf.h" 13 #include "btree_journal_iter.h" 14 #include "btree_key_cache.h" 15 #include "btree_locking.h" 16 #include "btree_node_scan.h" 17 #include "btree_update_interior.h" 18 #include "btree_io.h" 19 #include "btree_gc.h" 20 #include "buckets.h" 21 #include "clock.h" 22 #include "debug.h" 23 #include "disk_accounting.h" 24 #include "ec.h" 25 #include "error.h" 26 #include "extents.h" 27 #include "journal.h" 28 #include "keylist.h" 29 #include "move.h" 30 #include "recovery_passes.h" 31 #include "reflink.h" 32 #include "replicas.h" 33 #include "super-io.h" 34 #include "trace.h" 35 36 #include <linux/slab.h> 37 #include <linux/bitops.h> 38 #include <linux/freezer.h> 39 #include <linux/kthread.h> 40 #include <linux/preempt.h> 41 #include <linux/rcupdate.h> 42 #include <linux/sched/task.h> 43 44 #define DROP_THIS_NODE 10 45 #define DROP_PREV_NODE 11 46 #define DID_FILL_FROM_SCAN 12 47 48 static const char * const bch2_gc_phase_strs[] = { 49 #define x(n) #n, 50 GC_PHASES() 51 #undef x 52 NULL 53 }; 54 55 void bch2_gc_pos_to_text(struct printbuf *out, struct gc_pos *p) 56 { 57 prt_str(out, bch2_gc_phase_strs[p->phase]); 58 prt_char(out, ' '); 59 bch2_btree_id_to_text(out, p->btree); 60 prt_printf(out, " l=%u ", p->level); 61 bch2_bpos_to_text(out, p->pos); 62 } 63 64 static struct bkey_s unsafe_bkey_s_c_to_s(struct bkey_s_c k) 65 { 66 return (struct bkey_s) {{{ 67 (struct bkey *) k.k, 68 (struct bch_val *) k.v 69 }}}; 70 } 71 72 static inline void __gc_pos_set(struct bch_fs *c, struct gc_pos new_pos) 73 { 74 preempt_disable(); 75 write_seqcount_begin(&c->gc_pos_lock); 76 c->gc_pos = new_pos; 77 write_seqcount_end(&c->gc_pos_lock); 78 preempt_enable(); 79 } 80 81 static inline void gc_pos_set(struct bch_fs *c, struct gc_pos new_pos) 82 { 83 BUG_ON(gc_pos_cmp(new_pos, c->gc_pos) < 0); 84 __gc_pos_set(c, new_pos); 85 } 86 87 static void btree_ptr_to_v2(struct btree *b, struct bkey_i_btree_ptr_v2 *dst) 88 { 89 switch (b->key.k.type) { 90 case KEY_TYPE_btree_ptr: { 91 struct bkey_i_btree_ptr *src = bkey_i_to_btree_ptr(&b->key); 92 93 dst->k.p = src->k.p; 94 dst->v.mem_ptr = 0; 95 dst->v.seq = b->data->keys.seq; 96 dst->v.sectors_written = 0; 97 dst->v.flags = 0; 98 dst->v.min_key = b->data->min_key; 99 set_bkey_val_bytes(&dst->k, sizeof(dst->v) + bkey_val_bytes(&src->k)); 100 memcpy(dst->v.start, src->v.start, bkey_val_bytes(&src->k)); 101 break; 102 } 103 case KEY_TYPE_btree_ptr_v2: 104 bkey_copy(&dst->k_i, &b->key); 105 break; 106 default: 107 BUG(); 108 } 109 } 110 111 static int set_node_min(struct bch_fs *c, struct btree *b, struct bpos new_min) 112 { 113 struct bkey_i_btree_ptr_v2 *new; 114 int ret; 115 116 if (c->opts.verbose) { 117 struct printbuf buf = PRINTBUF; 118 119 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); 120 prt_str(&buf, " -> "); 121 bch2_bpos_to_text(&buf, new_min); 122 123 bch_info(c, "%s(): %s", __func__, buf.buf); 124 printbuf_exit(&buf); 125 } 126 127 new = kmalloc_array(BKEY_BTREE_PTR_U64s_MAX, sizeof(u64), GFP_KERNEL); 128 if (!new) 129 return -BCH_ERR_ENOMEM_gc_repair_key; 130 131 btree_ptr_to_v2(b, new); 132 b->data->min_key = new_min; 133 new->v.min_key = new_min; 134 SET_BTREE_PTR_RANGE_UPDATED(&new->v, true); 135 136 ret = bch2_journal_key_insert_take(c, b->c.btree_id, b->c.level + 1, &new->k_i); 137 if (ret) { 138 kfree(new); 139 return ret; 140 } 141 142 bch2_btree_node_drop_keys_outside_node(b); 143 bkey_copy(&b->key, &new->k_i); 144 return 0; 145 } 146 147 static int set_node_max(struct bch_fs *c, struct btree *b, struct bpos new_max) 148 { 149 struct bkey_i_btree_ptr_v2 *new; 150 int ret; 151 152 if (c->opts.verbose) { 153 struct printbuf buf = PRINTBUF; 154 155 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); 156 prt_str(&buf, " -> "); 157 bch2_bpos_to_text(&buf, new_max); 158 159 bch_info(c, "%s(): %s", __func__, buf.buf); 160 printbuf_exit(&buf); 161 } 162 163 ret = bch2_journal_key_delete(c, b->c.btree_id, b->c.level + 1, b->key.k.p); 164 if (ret) 165 return ret; 166 167 new = kmalloc_array(BKEY_BTREE_PTR_U64s_MAX, sizeof(u64), GFP_KERNEL); 168 if (!new) 169 return -BCH_ERR_ENOMEM_gc_repair_key; 170 171 btree_ptr_to_v2(b, new); 172 b->data->max_key = new_max; 173 new->k.p = new_max; 174 SET_BTREE_PTR_RANGE_UPDATED(&new->v, true); 175 176 ret = bch2_journal_key_insert_take(c, b->c.btree_id, b->c.level + 1, &new->k_i); 177 if (ret) { 178 kfree(new); 179 return ret; 180 } 181 182 bch2_btree_node_drop_keys_outside_node(b); 183 184 mutex_lock(&c->btree_cache.lock); 185 bch2_btree_node_hash_remove(&c->btree_cache, b); 186 187 bkey_copy(&b->key, &new->k_i); 188 ret = __bch2_btree_node_hash_insert(&c->btree_cache, b); 189 BUG_ON(ret); 190 mutex_unlock(&c->btree_cache.lock); 191 return 0; 192 } 193 194 static int btree_check_node_boundaries(struct btree_trans *trans, struct btree *b, 195 struct btree *prev, struct btree *cur, 196 struct bpos *pulled_from_scan) 197 { 198 struct bch_fs *c = trans->c; 199 struct bpos expected_start = !prev 200 ? b->data->min_key 201 : bpos_successor(prev->key.k.p); 202 struct printbuf buf = PRINTBUF; 203 int ret = 0; 204 205 BUG_ON(b->key.k.type == KEY_TYPE_btree_ptr_v2 && 206 !bpos_eq(bkey_i_to_btree_ptr_v2(&b->key)->v.min_key, 207 b->data->min_key)); 208 209 if (bpos_eq(expected_start, cur->data->min_key)) 210 return 0; 211 212 prt_printf(&buf, " at btree %s level %u:\n parent: ", 213 bch2_btree_id_str(b->c.btree_id), b->c.level); 214 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); 215 216 if (prev) { 217 prt_printf(&buf, "\n prev: "); 218 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&prev->key)); 219 } 220 221 prt_str(&buf, "\n next: "); 222 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&cur->key)); 223 224 if (bpos_lt(expected_start, cur->data->min_key)) { /* gap */ 225 if (b->c.level == 1 && 226 bpos_lt(*pulled_from_scan, cur->data->min_key)) { 227 ret = bch2_get_scanned_nodes(c, b->c.btree_id, 0, 228 expected_start, 229 bpos_predecessor(cur->data->min_key)); 230 if (ret) 231 goto err; 232 233 *pulled_from_scan = cur->data->min_key; 234 ret = DID_FILL_FROM_SCAN; 235 } else { 236 if (mustfix_fsck_err(trans, btree_node_topology_bad_min_key, 237 "btree node with incorrect min_key%s", buf.buf)) 238 ret = set_node_min(c, cur, expected_start); 239 } 240 } else { /* overlap */ 241 if (prev && BTREE_NODE_SEQ(cur->data) > BTREE_NODE_SEQ(prev->data)) { /* cur overwrites prev */ 242 if (bpos_ge(prev->data->min_key, cur->data->min_key)) { /* fully? */ 243 if (mustfix_fsck_err(trans, btree_node_topology_overwritten_by_next_node, 244 "btree node overwritten by next node%s", buf.buf)) 245 ret = DROP_PREV_NODE; 246 } else { 247 if (mustfix_fsck_err(trans, btree_node_topology_bad_max_key, 248 "btree node with incorrect max_key%s", buf.buf)) 249 ret = set_node_max(c, prev, 250 bpos_predecessor(cur->data->min_key)); 251 } 252 } else { 253 if (bpos_ge(expected_start, cur->data->max_key)) { /* fully? */ 254 if (mustfix_fsck_err(trans, btree_node_topology_overwritten_by_prev_node, 255 "btree node overwritten by prev node%s", buf.buf)) 256 ret = DROP_THIS_NODE; 257 } else { 258 if (mustfix_fsck_err(trans, btree_node_topology_bad_min_key, 259 "btree node with incorrect min_key%s", buf.buf)) 260 ret = set_node_min(c, cur, expected_start); 261 } 262 } 263 } 264 err: 265 fsck_err: 266 printbuf_exit(&buf); 267 return ret; 268 } 269 270 static int btree_repair_node_end(struct btree_trans *trans, struct btree *b, 271 struct btree *child, struct bpos *pulled_from_scan) 272 { 273 struct bch_fs *c = trans->c; 274 struct printbuf buf = PRINTBUF; 275 int ret = 0; 276 277 if (bpos_eq(child->key.k.p, b->key.k.p)) 278 return 0; 279 280 prt_printf(&buf, "at btree %s level %u:\n parent: ", 281 bch2_btree_id_str(b->c.btree_id), b->c.level); 282 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); 283 284 prt_str(&buf, "\n child: "); 285 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&child->key)); 286 287 if (mustfix_fsck_err(trans, btree_node_topology_bad_max_key, 288 "btree node with incorrect max_key%s", buf.buf)) { 289 if (b->c.level == 1 && 290 bpos_lt(*pulled_from_scan, b->key.k.p)) { 291 ret = bch2_get_scanned_nodes(c, b->c.btree_id, 0, 292 bpos_successor(child->key.k.p), b->key.k.p); 293 if (ret) 294 goto err; 295 296 *pulled_from_scan = b->key.k.p; 297 ret = DID_FILL_FROM_SCAN; 298 } else { 299 ret = set_node_max(c, child, b->key.k.p); 300 } 301 } 302 err: 303 fsck_err: 304 printbuf_exit(&buf); 305 return ret; 306 } 307 308 static int bch2_btree_repair_topology_recurse(struct btree_trans *trans, struct btree *b, 309 struct bpos *pulled_from_scan) 310 { 311 struct bch_fs *c = trans->c; 312 struct btree_and_journal_iter iter; 313 struct bkey_s_c k; 314 struct bkey_buf prev_k, cur_k; 315 struct btree *prev = NULL, *cur = NULL; 316 bool have_child, new_pass = false; 317 struct printbuf buf = PRINTBUF; 318 int ret = 0; 319 320 if (!b->c.level) 321 return 0; 322 323 bch2_bkey_buf_init(&prev_k); 324 bch2_bkey_buf_init(&cur_k); 325 again: 326 cur = prev = NULL; 327 have_child = new_pass = false; 328 bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); 329 iter.prefetch = true; 330 331 while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) { 332 BUG_ON(bpos_lt(k.k->p, b->data->min_key)); 333 BUG_ON(bpos_gt(k.k->p, b->data->max_key)); 334 335 bch2_btree_and_journal_iter_advance(&iter); 336 bch2_bkey_buf_reassemble(&cur_k, c, k); 337 338 cur = bch2_btree_node_get_noiter(trans, cur_k.k, 339 b->c.btree_id, b->c.level - 1, 340 false); 341 ret = PTR_ERR_OR_ZERO(cur); 342 343 printbuf_reset(&buf); 344 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(cur_k.k)); 345 346 if (mustfix_fsck_err_on(bch2_err_matches(ret, EIO), 347 trans, btree_node_unreadable, 348 "Topology repair: unreadable btree node at btree %s level %u:\n" 349 " %s", 350 bch2_btree_id_str(b->c.btree_id), 351 b->c.level - 1, 352 buf.buf)) { 353 bch2_btree_node_evict(trans, cur_k.k); 354 cur = NULL; 355 ret = bch2_journal_key_delete(c, b->c.btree_id, 356 b->c.level, cur_k.k->k.p); 357 if (ret) 358 break; 359 360 if (!btree_id_is_alloc(b->c.btree_id)) { 361 ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_scan_for_btree_nodes); 362 if (ret) 363 break; 364 } 365 continue; 366 } 367 368 bch_err_msg(c, ret, "getting btree node"); 369 if (ret) 370 break; 371 372 if (bch2_btree_node_is_stale(c, cur)) { 373 bch_info(c, "btree node %s older than nodes found by scanning", buf.buf); 374 six_unlock_read(&cur->c.lock); 375 bch2_btree_node_evict(trans, cur_k.k); 376 ret = bch2_journal_key_delete(c, b->c.btree_id, 377 b->c.level, cur_k.k->k.p); 378 cur = NULL; 379 if (ret) 380 break; 381 continue; 382 } 383 384 ret = btree_check_node_boundaries(trans, b, prev, cur, pulled_from_scan); 385 if (ret == DID_FILL_FROM_SCAN) { 386 new_pass = true; 387 ret = 0; 388 } 389 390 if (ret == DROP_THIS_NODE) { 391 six_unlock_read(&cur->c.lock); 392 bch2_btree_node_evict(trans, cur_k.k); 393 ret = bch2_journal_key_delete(c, b->c.btree_id, 394 b->c.level, cur_k.k->k.p); 395 cur = NULL; 396 if (ret) 397 break; 398 continue; 399 } 400 401 if (prev) 402 six_unlock_read(&prev->c.lock); 403 prev = NULL; 404 405 if (ret == DROP_PREV_NODE) { 406 bch_info(c, "dropped prev node"); 407 bch2_btree_node_evict(trans, prev_k.k); 408 ret = bch2_journal_key_delete(c, b->c.btree_id, 409 b->c.level, prev_k.k->k.p); 410 if (ret) 411 break; 412 413 bch2_btree_and_journal_iter_exit(&iter); 414 goto again; 415 } else if (ret) 416 break; 417 418 prev = cur; 419 cur = NULL; 420 bch2_bkey_buf_copy(&prev_k, c, cur_k.k); 421 } 422 423 if (!ret && !IS_ERR_OR_NULL(prev)) { 424 BUG_ON(cur); 425 ret = btree_repair_node_end(trans, b, prev, pulled_from_scan); 426 if (ret == DID_FILL_FROM_SCAN) { 427 new_pass = true; 428 ret = 0; 429 } 430 } 431 432 if (!IS_ERR_OR_NULL(prev)) 433 six_unlock_read(&prev->c.lock); 434 prev = NULL; 435 if (!IS_ERR_OR_NULL(cur)) 436 six_unlock_read(&cur->c.lock); 437 cur = NULL; 438 439 if (ret) 440 goto err; 441 442 bch2_btree_and_journal_iter_exit(&iter); 443 444 if (new_pass) 445 goto again; 446 447 bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); 448 iter.prefetch = true; 449 450 while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) { 451 bch2_bkey_buf_reassemble(&cur_k, c, k); 452 bch2_btree_and_journal_iter_advance(&iter); 453 454 cur = bch2_btree_node_get_noiter(trans, cur_k.k, 455 b->c.btree_id, b->c.level - 1, 456 false); 457 ret = PTR_ERR_OR_ZERO(cur); 458 459 bch_err_msg(c, ret, "getting btree node"); 460 if (ret) 461 goto err; 462 463 ret = bch2_btree_repair_topology_recurse(trans, cur, pulled_from_scan); 464 six_unlock_read(&cur->c.lock); 465 cur = NULL; 466 467 if (ret == DROP_THIS_NODE) { 468 bch2_btree_node_evict(trans, cur_k.k); 469 ret = bch2_journal_key_delete(c, b->c.btree_id, 470 b->c.level, cur_k.k->k.p); 471 new_pass = true; 472 } 473 474 if (ret) 475 goto err; 476 477 have_child = true; 478 } 479 480 printbuf_reset(&buf); 481 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); 482 483 if (mustfix_fsck_err_on(!have_child, 484 trans, btree_node_topology_interior_node_empty, 485 "empty interior btree node at btree %s level %u\n" 486 " %s", 487 bch2_btree_id_str(b->c.btree_id), 488 b->c.level, buf.buf)) 489 ret = DROP_THIS_NODE; 490 err: 491 fsck_err: 492 if (!IS_ERR_OR_NULL(prev)) 493 six_unlock_read(&prev->c.lock); 494 if (!IS_ERR_OR_NULL(cur)) 495 six_unlock_read(&cur->c.lock); 496 497 bch2_btree_and_journal_iter_exit(&iter); 498 499 if (!ret && new_pass) 500 goto again; 501 502 BUG_ON(!ret && bch2_btree_node_check_topology(trans, b)); 503 504 bch2_bkey_buf_exit(&prev_k, c); 505 bch2_bkey_buf_exit(&cur_k, c); 506 printbuf_exit(&buf); 507 return ret; 508 } 509 510 int bch2_check_topology(struct bch_fs *c) 511 { 512 struct btree_trans *trans = bch2_trans_get(c); 513 struct bpos pulled_from_scan = POS_MIN; 514 int ret = 0; 515 516 for (unsigned i = 0; i < btree_id_nr_alive(c) && !ret; i++) { 517 struct btree_root *r = bch2_btree_id_root(c, i); 518 bool reconstructed_root = false; 519 520 if (r->error) { 521 ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_scan_for_btree_nodes); 522 if (ret) 523 break; 524 reconstruct_root: 525 bch_info(c, "btree root %s unreadable, must recover from scan", bch2_btree_id_str(i)); 526 527 r->alive = false; 528 r->error = 0; 529 530 if (!bch2_btree_has_scanned_nodes(c, i)) { 531 mustfix_fsck_err(trans, btree_root_unreadable_and_scan_found_nothing, 532 "no nodes found for btree %s, continue?", bch2_btree_id_str(i)); 533 bch2_btree_root_alloc_fake_trans(trans, i, 0); 534 } else { 535 bch2_btree_root_alloc_fake_trans(trans, i, 1); 536 bch2_shoot_down_journal_keys(c, i, 1, BTREE_MAX_DEPTH, POS_MIN, SPOS_MAX); 537 ret = bch2_get_scanned_nodes(c, i, 0, POS_MIN, SPOS_MAX); 538 if (ret) 539 break; 540 } 541 542 reconstructed_root = true; 543 } 544 545 struct btree *b = r->b; 546 547 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); 548 ret = bch2_btree_repair_topology_recurse(trans, b, &pulled_from_scan); 549 six_unlock_read(&b->c.lock); 550 551 if (ret == DROP_THIS_NODE) { 552 bch2_btree_node_hash_remove(&c->btree_cache, b); 553 mutex_lock(&c->btree_cache.lock); 554 list_move(&b->list, &c->btree_cache.freeable); 555 mutex_unlock(&c->btree_cache.lock); 556 557 r->b = NULL; 558 559 if (!reconstructed_root) 560 goto reconstruct_root; 561 562 bch_err(c, "empty btree root %s", bch2_btree_id_str(i)); 563 bch2_btree_root_alloc_fake_trans(trans, i, 0); 564 r->alive = false; 565 ret = 0; 566 } 567 } 568 fsck_err: 569 bch2_trans_put(trans); 570 return ret; 571 } 572 573 /* marking of btree keys/nodes: */ 574 575 static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, 576 unsigned level, struct btree **prev, 577 struct btree_iter *iter, struct bkey_s_c k, 578 bool initial) 579 { 580 struct bch_fs *c = trans->c; 581 582 if (iter) { 583 struct btree_path *path = btree_iter_path(trans, iter); 584 struct btree *b = path_l(path)->b; 585 586 if (*prev != b) { 587 int ret = bch2_btree_node_check_topology(trans, b); 588 if (ret) 589 return ret; 590 } 591 *prev = b; 592 } 593 594 struct bkey deleted = KEY(0, 0, 0); 595 struct bkey_s_c old = (struct bkey_s_c) { &deleted, NULL }; 596 struct printbuf buf = PRINTBUF; 597 int ret = 0; 598 599 deleted.p = k.k->p; 600 601 if (initial) { 602 BUG_ON(bch2_journal_seq_verify && 603 k.k->version.lo > atomic64_read(&c->journal.seq)); 604 605 if (fsck_err_on(btree_id != BTREE_ID_accounting && 606 k.k->version.lo > atomic64_read(&c->key_version), 607 trans, bkey_version_in_future, 608 "key version number higher than recorded %llu\n %s", 609 atomic64_read(&c->key_version), 610 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 611 atomic64_set(&c->key_version, k.k->version.lo); 612 } 613 614 if (mustfix_fsck_err_on(level && !bch2_dev_btree_bitmap_marked(c, k), 615 trans, btree_bitmap_not_marked, 616 "btree ptr not marked in member info btree allocated bitmap\n %s", 617 (printbuf_reset(&buf), 618 bch2_bkey_val_to_text(&buf, c, k), 619 buf.buf))) { 620 mutex_lock(&c->sb_lock); 621 bch2_dev_btree_bitmap_mark(c, k); 622 bch2_write_super(c); 623 mutex_unlock(&c->sb_lock); 624 } 625 626 /* 627 * We require a commit before key_trigger() because 628 * key_trigger(BTREE_TRIGGER_GC) is not idempotant; we'll calculate the 629 * wrong result if we run it multiple times. 630 */ 631 unsigned flags = !iter ? BTREE_TRIGGER_is_root : 0; 632 633 ret = bch2_key_trigger(trans, btree_id, level, old, unsafe_bkey_s_c_to_s(k), 634 BTREE_TRIGGER_check_repair|flags); 635 if (ret) 636 goto out; 637 638 if (trans->nr_updates) { 639 ret = bch2_trans_commit(trans, NULL, NULL, 0) ?: 640 -BCH_ERR_transaction_restart_nested; 641 goto out; 642 } 643 644 ret = bch2_key_trigger(trans, btree_id, level, old, unsafe_bkey_s_c_to_s(k), 645 BTREE_TRIGGER_gc|BTREE_TRIGGER_insert|flags); 646 out: 647 fsck_err: 648 printbuf_exit(&buf); 649 bch_err_fn(c, ret); 650 return ret; 651 } 652 653 static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree, bool initial) 654 { 655 struct bch_fs *c = trans->c; 656 unsigned target_depth = btree_node_type_has_triggers(__btree_node_type(0, btree)) ? 0 : 1; 657 int ret = 0; 658 659 /* We need to make sure every leaf node is readable before going RW */ 660 if (initial) 661 target_depth = 0; 662 663 for (unsigned level = target_depth; level < BTREE_MAX_DEPTH; level++) { 664 struct btree *prev = NULL; 665 struct btree_iter iter; 666 bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 0, level, 667 BTREE_ITER_prefetch); 668 669 ret = for_each_btree_key_continue(trans, iter, 0, k, ({ 670 gc_pos_set(c, gc_pos_btree(btree, level, k.k->p)); 671 bch2_gc_mark_key(trans, btree, level, &prev, &iter, k, initial); 672 })); 673 if (ret) 674 goto err; 675 } 676 677 /* root */ 678 do { 679 retry_root: 680 bch2_trans_begin(trans); 681 682 struct btree_iter iter; 683 bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 684 0, bch2_btree_id_root(c, btree)->b->c.level, 0); 685 struct btree *b = bch2_btree_iter_peek_node(&iter); 686 ret = PTR_ERR_OR_ZERO(b); 687 if (ret) 688 goto err_root; 689 690 if (b != btree_node_root(c, b)) { 691 bch2_trans_iter_exit(trans, &iter); 692 goto retry_root; 693 } 694 695 gc_pos_set(c, gc_pos_btree(btree, b->c.level + 1, SPOS_MAX)); 696 struct bkey_s_c k = bkey_i_to_s_c(&b->key); 697 ret = bch2_gc_mark_key(trans, btree, b->c.level + 1, NULL, NULL, k, initial); 698 err_root: 699 bch2_trans_iter_exit(trans, &iter); 700 } while (bch2_err_matches(ret, BCH_ERR_transaction_restart)); 701 err: 702 bch_err_fn(c, ret); 703 return ret; 704 } 705 706 static inline int btree_id_gc_phase_cmp(enum btree_id l, enum btree_id r) 707 { 708 return cmp_int(gc_btree_order(l), gc_btree_order(r)); 709 } 710 711 static int bch2_gc_btrees(struct bch_fs *c) 712 { 713 struct btree_trans *trans = bch2_trans_get(c); 714 enum btree_id ids[BTREE_ID_NR]; 715 unsigned i; 716 int ret = 0; 717 718 for (i = 0; i < BTREE_ID_NR; i++) 719 ids[i] = i; 720 bubble_sort(ids, BTREE_ID_NR, btree_id_gc_phase_cmp); 721 722 for (i = 0; i < btree_id_nr_alive(c) && !ret; i++) { 723 unsigned btree = i < BTREE_ID_NR ? ids[i] : i; 724 725 if (IS_ERR_OR_NULL(bch2_btree_id_root(c, btree)->b)) 726 continue; 727 728 ret = bch2_gc_btree(trans, btree, true); 729 730 if (mustfix_fsck_err_on(bch2_err_matches(ret, EIO), 731 trans, btree_node_read_error, 732 "btree node read error for %s", 733 bch2_btree_id_str(btree))) 734 ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology); 735 } 736 fsck_err: 737 bch2_trans_put(trans); 738 bch_err_fn(c, ret); 739 return ret; 740 } 741 742 static int bch2_mark_superblocks(struct bch_fs *c) 743 { 744 mutex_lock(&c->sb_lock); 745 gc_pos_set(c, gc_phase(GC_PHASE_sb)); 746 747 int ret = bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_gc); 748 mutex_unlock(&c->sb_lock); 749 return ret; 750 } 751 752 static void bch2_gc_free(struct bch_fs *c) 753 { 754 bch2_accounting_gc_free(c); 755 756 genradix_free(&c->reflink_gc_table); 757 genradix_free(&c->gc_stripes); 758 759 for_each_member_device(c, ca) { 760 kvfree(rcu_dereference_protected(ca->buckets_gc, 1)); 761 ca->buckets_gc = NULL; 762 } 763 } 764 765 static int bch2_gc_start(struct bch_fs *c) 766 { 767 for_each_member_device(c, ca) { 768 int ret = bch2_dev_usage_init(ca, true); 769 if (ret) { 770 bch2_dev_put(ca); 771 return ret; 772 } 773 } 774 775 return 0; 776 } 777 778 /* returns true if not equal */ 779 static inline bool bch2_alloc_v4_cmp(struct bch_alloc_v4 l, 780 struct bch_alloc_v4 r) 781 { 782 return l.gen != r.gen || 783 l.oldest_gen != r.oldest_gen || 784 l.data_type != r.data_type || 785 l.dirty_sectors != r.dirty_sectors || 786 l.stripe_sectors != r.stripe_sectors || 787 l.cached_sectors != r.cached_sectors || 788 l.stripe_redundancy != r.stripe_redundancy || 789 l.stripe != r.stripe; 790 } 791 792 static int bch2_alloc_write_key(struct btree_trans *trans, 793 struct btree_iter *iter, 794 struct bch_dev *ca, 795 struct bkey_s_c k) 796 { 797 struct bch_fs *c = trans->c; 798 struct bkey_i_alloc_v4 *a; 799 struct bch_alloc_v4 old_gc, gc, old_convert, new; 800 const struct bch_alloc_v4 *old; 801 int ret; 802 803 if (!bucket_valid(ca, k.k->p.offset)) 804 return 0; 805 806 old = bch2_alloc_to_v4(k, &old_convert); 807 gc = new = *old; 808 809 percpu_down_read(&c->mark_lock); 810 __bucket_m_to_alloc(&gc, *gc_bucket(ca, iter->pos.offset)); 811 812 old_gc = gc; 813 814 if ((old->data_type == BCH_DATA_sb || 815 old->data_type == BCH_DATA_journal) && 816 !bch2_dev_is_online(ca)) { 817 gc.data_type = old->data_type; 818 gc.dirty_sectors = old->dirty_sectors; 819 } 820 percpu_up_read(&c->mark_lock); 821 822 /* 823 * gc.data_type doesn't yet include need_discard & need_gc_gen states - 824 * fix that here: 825 */ 826 alloc_data_type_set(&gc, gc.data_type); 827 828 if (gc.data_type != old_gc.data_type || 829 gc.dirty_sectors != old_gc.dirty_sectors) { 830 ret = bch2_alloc_key_to_dev_counters(trans, ca, &old_gc, &gc, BTREE_TRIGGER_gc); 831 if (ret) 832 return ret; 833 } 834 835 gc.fragmentation_lru = alloc_lru_idx_fragmentation(gc, ca); 836 837 if (fsck_err_on(new.data_type != gc.data_type, 838 trans, alloc_key_data_type_wrong, 839 "bucket %llu:%llu gen %u has wrong data_type" 840 ": got %s, should be %s", 841 iter->pos.inode, iter->pos.offset, 842 gc.gen, 843 bch2_data_type_str(new.data_type), 844 bch2_data_type_str(gc.data_type))) 845 new.data_type = gc.data_type; 846 847 #define copy_bucket_field(_errtype, _f) \ 848 if (fsck_err_on(new._f != gc._f, \ 849 trans, _errtype, \ 850 "bucket %llu:%llu gen %u data type %s has wrong " #_f \ 851 ": got %llu, should be %llu", \ 852 iter->pos.inode, iter->pos.offset, \ 853 gc.gen, \ 854 bch2_data_type_str(gc.data_type), \ 855 (u64) new._f, (u64) gc._f)) \ 856 new._f = gc._f; \ 857 858 copy_bucket_field(alloc_key_gen_wrong, gen); 859 copy_bucket_field(alloc_key_dirty_sectors_wrong, dirty_sectors); 860 copy_bucket_field(alloc_key_stripe_sectors_wrong, stripe_sectors); 861 copy_bucket_field(alloc_key_cached_sectors_wrong, cached_sectors); 862 copy_bucket_field(alloc_key_stripe_wrong, stripe); 863 copy_bucket_field(alloc_key_stripe_redundancy_wrong, stripe_redundancy); 864 copy_bucket_field(alloc_key_fragmentation_lru_wrong, fragmentation_lru); 865 #undef copy_bucket_field 866 867 if (!bch2_alloc_v4_cmp(*old, new)) 868 return 0; 869 870 a = bch2_alloc_to_v4_mut(trans, k); 871 ret = PTR_ERR_OR_ZERO(a); 872 if (ret) 873 return ret; 874 875 a->v = new; 876 877 /* 878 * The trigger normally makes sure these are set, but we're not running 879 * triggers: 880 */ 881 if (a->v.data_type == BCH_DATA_cached && !a->v.io_time[READ]) 882 a->v.io_time[READ] = max_t(u64, 1, atomic64_read(&c->io_clock[READ].now)); 883 884 ret = bch2_trans_update(trans, iter, &a->k_i, BTREE_TRIGGER_norun); 885 fsck_err: 886 return ret; 887 } 888 889 static int bch2_gc_alloc_done(struct bch_fs *c) 890 { 891 int ret = 0; 892 893 for_each_member_device(c, ca) { 894 ret = bch2_trans_run(c, 895 for_each_btree_key_upto_commit(trans, iter, BTREE_ID_alloc, 896 POS(ca->dev_idx, ca->mi.first_bucket), 897 POS(ca->dev_idx, ca->mi.nbuckets - 1), 898 BTREE_ITER_slots|BTREE_ITER_prefetch, k, 899 NULL, NULL, BCH_TRANS_COMMIT_lazy_rw, 900 bch2_alloc_write_key(trans, &iter, ca, k))); 901 if (ret) { 902 bch2_dev_put(ca); 903 break; 904 } 905 } 906 907 bch_err_fn(c, ret); 908 return ret; 909 } 910 911 static int bch2_gc_alloc_start(struct bch_fs *c) 912 { 913 int ret = 0; 914 915 for_each_member_device(c, ca) { 916 struct bucket_array *buckets = kvmalloc(sizeof(struct bucket_array) + 917 ca->mi.nbuckets * sizeof(struct bucket), 918 GFP_KERNEL|__GFP_ZERO); 919 if (!buckets) { 920 bch2_dev_put(ca); 921 ret = -BCH_ERR_ENOMEM_gc_alloc_start; 922 break; 923 } 924 925 buckets->first_bucket = ca->mi.first_bucket; 926 buckets->nbuckets = ca->mi.nbuckets; 927 buckets->nbuckets_minus_first = 928 buckets->nbuckets - buckets->first_bucket; 929 rcu_assign_pointer(ca->buckets_gc, buckets); 930 } 931 932 bch_err_fn(c, ret); 933 return ret; 934 } 935 936 static int bch2_gc_write_reflink_key(struct btree_trans *trans, 937 struct btree_iter *iter, 938 struct bkey_s_c k, 939 size_t *idx) 940 { 941 struct bch_fs *c = trans->c; 942 const __le64 *refcount = bkey_refcount_c(k); 943 struct printbuf buf = PRINTBUF; 944 struct reflink_gc *r; 945 int ret = 0; 946 947 if (!refcount) 948 return 0; 949 950 while ((r = genradix_ptr(&c->reflink_gc_table, *idx)) && 951 r->offset < k.k->p.offset) 952 ++*idx; 953 954 if (!r || 955 r->offset != k.k->p.offset || 956 r->size != k.k->size) { 957 bch_err(c, "unexpected inconsistency walking reflink table at gc finish"); 958 return -EINVAL; 959 } 960 961 if (fsck_err_on(r->refcount != le64_to_cpu(*refcount), 962 trans, reflink_v_refcount_wrong, 963 "reflink key has wrong refcount:\n" 964 " %s\n" 965 " should be %u", 966 (bch2_bkey_val_to_text(&buf, c, k), buf.buf), 967 r->refcount)) { 968 struct bkey_i *new = bch2_bkey_make_mut_noupdate(trans, k); 969 ret = PTR_ERR_OR_ZERO(new); 970 if (ret) 971 goto out; 972 973 if (!r->refcount) 974 new->k.type = KEY_TYPE_deleted; 975 else 976 *bkey_refcount(bkey_i_to_s(new)) = cpu_to_le64(r->refcount); 977 ret = bch2_trans_update(trans, iter, new, 0); 978 } 979 out: 980 fsck_err: 981 printbuf_exit(&buf); 982 return ret; 983 } 984 985 static int bch2_gc_reflink_done(struct bch_fs *c) 986 { 987 size_t idx = 0; 988 989 int ret = bch2_trans_run(c, 990 for_each_btree_key_commit(trans, iter, 991 BTREE_ID_reflink, POS_MIN, 992 BTREE_ITER_prefetch, k, 993 NULL, NULL, BCH_TRANS_COMMIT_no_enospc, 994 bch2_gc_write_reflink_key(trans, &iter, k, &idx))); 995 c->reflink_gc_nr = 0; 996 return ret; 997 } 998 999 static int bch2_gc_reflink_start(struct bch_fs *c) 1000 { 1001 c->reflink_gc_nr = 0; 1002 1003 int ret = bch2_trans_run(c, 1004 for_each_btree_key(trans, iter, BTREE_ID_reflink, POS_MIN, 1005 BTREE_ITER_prefetch, k, ({ 1006 const __le64 *refcount = bkey_refcount_c(k); 1007 1008 if (!refcount) 1009 continue; 1010 1011 struct reflink_gc *r = genradix_ptr_alloc(&c->reflink_gc_table, 1012 c->reflink_gc_nr++, GFP_KERNEL); 1013 if (!r) { 1014 ret = -BCH_ERR_ENOMEM_gc_reflink_start; 1015 break; 1016 } 1017 1018 r->offset = k.k->p.offset; 1019 r->size = k.k->size; 1020 r->refcount = 0; 1021 0; 1022 }))); 1023 1024 bch_err_fn(c, ret); 1025 return ret; 1026 } 1027 1028 static int bch2_gc_write_stripes_key(struct btree_trans *trans, 1029 struct btree_iter *iter, 1030 struct bkey_s_c k) 1031 { 1032 struct bch_fs *c = trans->c; 1033 struct printbuf buf = PRINTBUF; 1034 const struct bch_stripe *s; 1035 struct gc_stripe *m; 1036 bool bad = false; 1037 unsigned i; 1038 int ret = 0; 1039 1040 if (k.k->type != KEY_TYPE_stripe) 1041 return 0; 1042 1043 s = bkey_s_c_to_stripe(k).v; 1044 m = genradix_ptr(&c->gc_stripes, k.k->p.offset); 1045 1046 for (i = 0; i < s->nr_blocks; i++) { 1047 u32 old = stripe_blockcount_get(s, i); 1048 u32 new = (m ? m->block_sectors[i] : 0); 1049 1050 if (old != new) { 1051 prt_printf(&buf, "stripe block %u has wrong sector count: got %u, should be %u\n", 1052 i, old, new); 1053 bad = true; 1054 } 1055 } 1056 1057 if (bad) 1058 bch2_bkey_val_to_text(&buf, c, k); 1059 1060 if (fsck_err_on(bad, 1061 trans, stripe_sector_count_wrong, 1062 "%s", buf.buf)) { 1063 struct bkey_i_stripe *new; 1064 1065 new = bch2_trans_kmalloc(trans, bkey_bytes(k.k)); 1066 ret = PTR_ERR_OR_ZERO(new); 1067 if (ret) 1068 return ret; 1069 1070 bkey_reassemble(&new->k_i, k); 1071 1072 for (i = 0; i < new->v.nr_blocks; i++) 1073 stripe_blockcount_set(&new->v, i, m ? m->block_sectors[i] : 0); 1074 1075 ret = bch2_trans_update(trans, iter, &new->k_i, 0); 1076 } 1077 fsck_err: 1078 printbuf_exit(&buf); 1079 return ret; 1080 } 1081 1082 static int bch2_gc_stripes_done(struct bch_fs *c) 1083 { 1084 return bch2_trans_run(c, 1085 for_each_btree_key_commit(trans, iter, 1086 BTREE_ID_stripes, POS_MIN, 1087 BTREE_ITER_prefetch, k, 1088 NULL, NULL, BCH_TRANS_COMMIT_no_enospc, 1089 bch2_gc_write_stripes_key(trans, &iter, k))); 1090 } 1091 1092 /** 1093 * bch2_check_allocations - walk all references to buckets, and recompute them: 1094 * 1095 * @c: filesystem object 1096 * 1097 * Returns: 0 on success, or standard errcode on failure 1098 * 1099 * Order matters here: 1100 * - Concurrent GC relies on the fact that we have a total ordering for 1101 * everything that GC walks - see gc_will_visit_node(), 1102 * gc_will_visit_root() 1103 * 1104 * - also, references move around in the course of index updates and 1105 * various other crap: everything needs to agree on the ordering 1106 * references are allowed to move around in - e.g., we're allowed to 1107 * start with a reference owned by an open_bucket (the allocator) and 1108 * move it to the btree, but not the reverse. 1109 * 1110 * This is necessary to ensure that gc doesn't miss references that 1111 * move around - if references move backwards in the ordering GC 1112 * uses, GC could skip past them 1113 */ 1114 int bch2_check_allocations(struct bch_fs *c) 1115 { 1116 int ret; 1117 1118 lockdep_assert_held(&c->state_lock); 1119 1120 down_write(&c->gc_lock); 1121 1122 bch2_btree_interior_updates_flush(c); 1123 1124 ret = bch2_gc_accounting_start(c) ?: 1125 bch2_gc_start(c) ?: 1126 bch2_gc_alloc_start(c) ?: 1127 bch2_gc_reflink_start(c); 1128 if (ret) 1129 goto out; 1130 1131 gc_pos_set(c, gc_phase(GC_PHASE_start)); 1132 1133 ret = bch2_mark_superblocks(c); 1134 bch_err_msg(c, ret, "marking superblocks"); 1135 if (ret) 1136 goto out; 1137 1138 ret = bch2_gc_btrees(c); 1139 if (ret) 1140 goto out; 1141 1142 c->gc_count++; 1143 1144 ret = bch2_gc_alloc_done(c) ?: 1145 bch2_gc_accounting_done(c) ?: 1146 bch2_gc_stripes_done(c) ?: 1147 bch2_gc_reflink_done(c); 1148 out: 1149 percpu_down_write(&c->mark_lock); 1150 /* Indicates that gc is no longer in progress: */ 1151 __gc_pos_set(c, gc_phase(GC_PHASE_not_running)); 1152 1153 bch2_gc_free(c); 1154 percpu_up_write(&c->mark_lock); 1155 1156 up_write(&c->gc_lock); 1157 1158 /* 1159 * At startup, allocations can happen directly instead of via the 1160 * allocator thread - issue wakeup in case they blocked on gc_lock: 1161 */ 1162 closure_wake_up(&c->freelist_wait); 1163 bch_err_fn(c, ret); 1164 return ret; 1165 } 1166 1167 static int gc_btree_gens_key(struct btree_trans *trans, 1168 struct btree_iter *iter, 1169 struct bkey_s_c k) 1170 { 1171 struct bch_fs *c = trans->c; 1172 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); 1173 struct bkey_i *u; 1174 int ret; 1175 1176 if (unlikely(test_bit(BCH_FS_going_ro, &c->flags))) 1177 return -EROFS; 1178 1179 percpu_down_read(&c->mark_lock); 1180 rcu_read_lock(); 1181 bkey_for_each_ptr(ptrs, ptr) { 1182 struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); 1183 if (!ca) 1184 continue; 1185 1186 if (dev_ptr_stale(ca, ptr) > 16) { 1187 rcu_read_unlock(); 1188 percpu_up_read(&c->mark_lock); 1189 goto update; 1190 } 1191 } 1192 1193 bkey_for_each_ptr(ptrs, ptr) { 1194 struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); 1195 if (!ca) 1196 continue; 1197 1198 u8 *gen = &ca->oldest_gen[PTR_BUCKET_NR(ca, ptr)]; 1199 if (gen_after(*gen, ptr->gen)) 1200 *gen = ptr->gen; 1201 } 1202 rcu_read_unlock(); 1203 percpu_up_read(&c->mark_lock); 1204 return 0; 1205 update: 1206 u = bch2_bkey_make_mut(trans, iter, &k, 0); 1207 ret = PTR_ERR_OR_ZERO(u); 1208 if (ret) 1209 return ret; 1210 1211 bch2_extent_normalize(c, bkey_i_to_s(u)); 1212 return 0; 1213 } 1214 1215 static int bch2_alloc_write_oldest_gen(struct btree_trans *trans, struct bch_dev *ca, 1216 struct btree_iter *iter, struct bkey_s_c k) 1217 { 1218 struct bch_alloc_v4 a_convert; 1219 const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert); 1220 struct bkey_i_alloc_v4 *a_mut; 1221 int ret; 1222 1223 if (a->oldest_gen == ca->oldest_gen[iter->pos.offset]) 1224 return 0; 1225 1226 a_mut = bch2_alloc_to_v4_mut(trans, k); 1227 ret = PTR_ERR_OR_ZERO(a_mut); 1228 if (ret) 1229 return ret; 1230 1231 a_mut->v.oldest_gen = ca->oldest_gen[iter->pos.offset]; 1232 alloc_data_type_set(&a_mut->v, a_mut->v.data_type); 1233 1234 return bch2_trans_update(trans, iter, &a_mut->k_i, 0); 1235 } 1236 1237 int bch2_gc_gens(struct bch_fs *c) 1238 { 1239 u64 b, start_time = local_clock(); 1240 int ret; 1241 1242 /* 1243 * Ideally we would be using state_lock and not gc_gens_lock here, but that 1244 * introduces a deadlock in the RO path - we currently take the state 1245 * lock at the start of going RO, thus the gc thread may get stuck: 1246 */ 1247 if (!mutex_trylock(&c->gc_gens_lock)) 1248 return 0; 1249 1250 trace_and_count(c, gc_gens_start, c); 1251 1252 down_read(&c->state_lock); 1253 1254 for_each_member_device(c, ca) { 1255 struct bucket_gens *gens = bucket_gens(ca); 1256 1257 BUG_ON(ca->oldest_gen); 1258 1259 ca->oldest_gen = kvmalloc(gens->nbuckets, GFP_KERNEL); 1260 if (!ca->oldest_gen) { 1261 bch2_dev_put(ca); 1262 ret = -BCH_ERR_ENOMEM_gc_gens; 1263 goto err; 1264 } 1265 1266 for (b = gens->first_bucket; 1267 b < gens->nbuckets; b++) 1268 ca->oldest_gen[b] = gens->b[b]; 1269 } 1270 1271 for (unsigned i = 0; i < BTREE_ID_NR; i++) 1272 if (btree_type_has_ptrs(i)) { 1273 c->gc_gens_btree = i; 1274 c->gc_gens_pos = POS_MIN; 1275 1276 ret = bch2_trans_run(c, 1277 for_each_btree_key_commit(trans, iter, i, 1278 POS_MIN, 1279 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, 1280 k, 1281 NULL, NULL, 1282 BCH_TRANS_COMMIT_no_enospc, 1283 gc_btree_gens_key(trans, &iter, k))); 1284 if (ret) 1285 goto err; 1286 } 1287 1288 struct bch_dev *ca = NULL; 1289 ret = bch2_trans_run(c, 1290 for_each_btree_key_commit(trans, iter, BTREE_ID_alloc, 1291 POS_MIN, 1292 BTREE_ITER_prefetch, 1293 k, 1294 NULL, NULL, 1295 BCH_TRANS_COMMIT_no_enospc, ({ 1296 ca = bch2_dev_iterate(c, ca, k.k->p.inode); 1297 if (!ca) { 1298 bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); 1299 continue; 1300 } 1301 bch2_alloc_write_oldest_gen(trans, ca, &iter, k); 1302 }))); 1303 bch2_dev_put(ca); 1304 1305 if (ret) 1306 goto err; 1307 1308 c->gc_gens_btree = 0; 1309 c->gc_gens_pos = POS_MIN; 1310 1311 c->gc_count++; 1312 1313 bch2_time_stats_update(&c->times[BCH_TIME_btree_gc], start_time); 1314 trace_and_count(c, gc_gens_end, c); 1315 err: 1316 for_each_member_device(c, ca) { 1317 kvfree(ca->oldest_gen); 1318 ca->oldest_gen = NULL; 1319 } 1320 1321 up_read(&c->state_lock); 1322 mutex_unlock(&c->gc_gens_lock); 1323 if (!bch2_err_matches(ret, EROFS)) 1324 bch_err_fn(c, ret); 1325 return ret; 1326 } 1327 1328 static void bch2_gc_gens_work(struct work_struct *work) 1329 { 1330 struct bch_fs *c = container_of(work, struct bch_fs, gc_gens_work); 1331 bch2_gc_gens(c); 1332 bch2_write_ref_put(c, BCH_WRITE_REF_gc_gens); 1333 } 1334 1335 void bch2_gc_gens_async(struct bch_fs *c) 1336 { 1337 if (bch2_write_ref_tryget(c, BCH_WRITE_REF_gc_gens) && 1338 !queue_work(c->write_ref_wq, &c->gc_gens_work)) 1339 bch2_write_ref_put(c, BCH_WRITE_REF_gc_gens); 1340 } 1341 1342 void bch2_fs_gc_init(struct bch_fs *c) 1343 { 1344 seqcount_init(&c->gc_pos_lock); 1345 1346 INIT_WORK(&c->gc_gens_work, bch2_gc_gens_work); 1347 } 1348