1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for manipulating bucket marks for garbage collection. 4 * 5 * Copyright 2014 Datera, Inc. 6 */ 7 8 #include "bcachefs.h" 9 #include "alloc_background.h" 10 #include "backpointers.h" 11 #include "bset.h" 12 #include "btree_gc.h" 13 #include "btree_update.h" 14 #include "buckets.h" 15 #include "buckets_waiting_for_journal.h" 16 #include "disk_accounting.h" 17 #include "ec.h" 18 #include "error.h" 19 #include "inode.h" 20 #include "movinggc.h" 21 #include "rebalance.h" 22 #include "recovery.h" 23 #include "recovery_passes.h" 24 #include "reflink.h" 25 #include "replicas.h" 26 #include "subvolume.h" 27 #include "trace.h" 28 29 #include <linux/preempt.h> 30 31 void bch2_dev_usage_read_fast(struct bch_dev *ca, struct bch_dev_usage *usage) 32 { 33 memset(usage, 0, sizeof(*usage)); 34 acc_u64s_percpu((u64 *) usage, (u64 __percpu *) ca->usage, dev_usage_u64s()); 35 } 36 37 static u64 reserve_factor(u64 r) 38 { 39 return r + (round_up(r, (1 << RESERVE_FACTOR)) >> RESERVE_FACTOR); 40 } 41 42 static struct bch_fs_usage_short 43 __bch2_fs_usage_read_short(struct bch_fs *c) 44 { 45 struct bch_fs_usage_short ret; 46 u64 data, reserved; 47 48 ret.capacity = c->capacity - 49 percpu_u64_get(&c->usage->hidden); 50 51 data = percpu_u64_get(&c->usage->data) + 52 percpu_u64_get(&c->usage->btree); 53 reserved = percpu_u64_get(&c->usage->reserved) + 54 percpu_u64_get(c->online_reserved); 55 56 ret.used = min(ret.capacity, data + reserve_factor(reserved)); 57 ret.free = ret.capacity - ret.used; 58 59 ret.nr_inodes = percpu_u64_get(&c->usage->nr_inodes); 60 61 return ret; 62 } 63 64 struct bch_fs_usage_short 65 bch2_fs_usage_read_short(struct bch_fs *c) 66 { 67 struct bch_fs_usage_short ret; 68 69 percpu_down_read(&c->mark_lock); 70 ret = __bch2_fs_usage_read_short(c); 71 percpu_up_read(&c->mark_lock); 72 73 return ret; 74 } 75 76 void bch2_dev_usage_to_text(struct printbuf *out, 77 struct bch_dev *ca, 78 struct bch_dev_usage *usage) 79 { 80 if (out->nr_tabstops < 5) { 81 printbuf_tabstops_reset(out); 82 printbuf_tabstop_push(out, 12); 83 printbuf_tabstop_push(out, 16); 84 printbuf_tabstop_push(out, 16); 85 printbuf_tabstop_push(out, 16); 86 printbuf_tabstop_push(out, 16); 87 } 88 89 prt_printf(out, "\tbuckets\rsectors\rfragmented\r\n"); 90 91 for (unsigned i = 0; i < BCH_DATA_NR; i++) { 92 bch2_prt_data_type(out, i); 93 prt_printf(out, "\t%llu\r%llu\r%llu\r\n", 94 usage->d[i].buckets, 95 usage->d[i].sectors, 96 usage->d[i].fragmented); 97 } 98 99 prt_printf(out, "capacity\t%llu\r\n", ca->mi.nbuckets); 100 } 101 102 static int bch2_check_fix_ptr(struct btree_trans *trans, 103 struct bkey_s_c k, 104 struct extent_ptr_decoded p, 105 const union bch_extent_entry *entry, 106 bool *do_update) 107 { 108 struct bch_fs *c = trans->c; 109 struct printbuf buf = PRINTBUF; 110 int ret = 0; 111 112 struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev); 113 if (!ca) { 114 if (fsck_err_on(p.ptr.dev != BCH_SB_MEMBER_INVALID, 115 trans, ptr_to_invalid_device, 116 "pointer to missing device %u\n" 117 "while marking %s", 118 p.ptr.dev, 119 (printbuf_reset(&buf), 120 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 121 *do_update = true; 122 return 0; 123 } 124 125 struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); 126 if (!g) { 127 if (fsck_err(trans, ptr_to_invalid_device, 128 "pointer to invalid bucket on device %u\n" 129 "while marking %s", 130 p.ptr.dev, 131 (printbuf_reset(&buf), 132 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 133 *do_update = true; 134 goto out; 135 } 136 137 enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry); 138 139 if (fsck_err_on(!g->gen_valid, 140 trans, ptr_to_missing_alloc_key, 141 "bucket %u:%zu data type %s ptr gen %u missing in alloc btree\n" 142 "while marking %s", 143 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), 144 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)), 145 p.ptr.gen, 146 (printbuf_reset(&buf), 147 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { 148 if (!p.ptr.cached) { 149 g->gen_valid = true; 150 g->gen = p.ptr.gen; 151 } else { 152 *do_update = true; 153 } 154 } 155 156 if (fsck_err_on(gen_cmp(p.ptr.gen, g->gen) > 0, 157 trans, ptr_gen_newer_than_bucket_gen, 158 "bucket %u:%zu data type %s ptr gen in the future: %u > %u\n" 159 "while marking %s", 160 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), 161 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)), 162 p.ptr.gen, g->gen, 163 (printbuf_reset(&buf), 164 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { 165 if (!p.ptr.cached && 166 (g->data_type != BCH_DATA_btree || 167 data_type == BCH_DATA_btree)) { 168 g->gen_valid = true; 169 g->gen = p.ptr.gen; 170 g->data_type = 0; 171 g->stripe_sectors = 0; 172 g->dirty_sectors = 0; 173 g->cached_sectors = 0; 174 } else { 175 *do_update = true; 176 } 177 } 178 179 if (fsck_err_on(gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX, 180 trans, ptr_gen_newer_than_bucket_gen, 181 "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n" 182 "while marking %s", 183 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, 184 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)), 185 p.ptr.gen, 186 (printbuf_reset(&buf), 187 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 188 *do_update = true; 189 190 if (fsck_err_on(!p.ptr.cached && gen_cmp(p.ptr.gen, g->gen) < 0, 191 trans, stale_dirty_ptr, 192 "bucket %u:%zu data type %s stale dirty ptr: %u < %u\n" 193 "while marking %s", 194 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), 195 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)), 196 p.ptr.gen, g->gen, 197 (printbuf_reset(&buf), 198 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 199 *do_update = true; 200 201 if (data_type != BCH_DATA_btree && p.ptr.gen != g->gen) 202 goto out; 203 204 if (fsck_err_on(bucket_data_type_mismatch(g->data_type, data_type), 205 trans, ptr_bucket_data_type_mismatch, 206 "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n" 207 "while marking %s", 208 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, 209 bch2_data_type_str(g->data_type), 210 bch2_data_type_str(data_type), 211 (printbuf_reset(&buf), 212 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { 213 if (data_type == BCH_DATA_btree) { 214 g->gen_valid = true; 215 g->gen = p.ptr.gen; 216 g->data_type = data_type; 217 g->stripe_sectors = 0; 218 g->dirty_sectors = 0; 219 g->cached_sectors = 0; 220 } else { 221 *do_update = true; 222 } 223 } 224 225 if (p.has_ec) { 226 struct gc_stripe *m = genradix_ptr(&c->gc_stripes, p.ec.idx); 227 228 if (fsck_err_on(!m || !m->alive, 229 trans, ptr_to_missing_stripe, 230 "pointer to nonexistent stripe %llu\n" 231 "while marking %s", 232 (u64) p.ec.idx, 233 (printbuf_reset(&buf), 234 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 235 *do_update = true; 236 237 if (fsck_err_on(m && m->alive && !bch2_ptr_matches_stripe_m(m, p), 238 trans, ptr_to_incorrect_stripe, 239 "pointer does not match stripe %llu\n" 240 "while marking %s", 241 (u64) p.ec.idx, 242 (printbuf_reset(&buf), 243 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 244 *do_update = true; 245 } 246 out: 247 fsck_err: 248 bch2_dev_put(ca); 249 printbuf_exit(&buf); 250 return ret; 251 } 252 253 int bch2_check_fix_ptrs(struct btree_trans *trans, 254 enum btree_id btree, unsigned level, struct bkey_s_c k, 255 enum btree_iter_update_trigger_flags flags) 256 { 257 struct bch_fs *c = trans->c; 258 struct bkey_ptrs_c ptrs_c = bch2_bkey_ptrs_c(k); 259 const union bch_extent_entry *entry_c; 260 struct extent_ptr_decoded p = { 0 }; 261 bool do_update = false; 262 struct printbuf buf = PRINTBUF; 263 int ret = 0; 264 265 bkey_for_each_ptr_decode(k.k, ptrs_c, p, entry_c) { 266 ret = bch2_check_fix_ptr(trans, k, p, entry_c, &do_update); 267 if (ret) 268 goto err; 269 } 270 271 if (do_update) { 272 if (flags & BTREE_TRIGGER_is_root) { 273 bch_err(c, "cannot update btree roots yet"); 274 ret = -EINVAL; 275 goto err; 276 } 277 278 struct bkey_i *new = bch2_bkey_make_mut_noupdate(trans, k); 279 ret = PTR_ERR_OR_ZERO(new); 280 if (ret) 281 goto err; 282 283 rcu_read_lock(); 284 bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, !bch2_dev_exists(c, ptr->dev)); 285 rcu_read_unlock(); 286 287 if (level) { 288 /* 289 * We don't want to drop btree node pointers - if the 290 * btree node isn't there anymore, the read path will 291 * sort it out: 292 */ 293 struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); 294 rcu_read_lock(); 295 bkey_for_each_ptr(ptrs, ptr) { 296 struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); 297 struct bucket *g = PTR_GC_BUCKET(ca, ptr); 298 299 ptr->gen = g->gen; 300 } 301 rcu_read_unlock(); 302 } else { 303 struct bkey_ptrs ptrs; 304 union bch_extent_entry *entry; 305 306 rcu_read_lock(); 307 restart_drop_ptrs: 308 ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); 309 bkey_for_each_ptr_decode(bkey_i_to_s(new).k, ptrs, p, entry) { 310 struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev); 311 struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); 312 enum bch_data_type data_type = bch2_bkey_ptr_data_type(bkey_i_to_s_c(new), p, entry); 313 314 if ((p.ptr.cached && 315 (!g->gen_valid || gen_cmp(p.ptr.gen, g->gen) > 0)) || 316 (!p.ptr.cached && 317 gen_cmp(p.ptr.gen, g->gen) < 0) || 318 gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX || 319 (g->data_type && 320 g->data_type != data_type)) { 321 bch2_bkey_drop_ptr(bkey_i_to_s(new), &entry->ptr); 322 goto restart_drop_ptrs; 323 } 324 } 325 rcu_read_unlock(); 326 again: 327 ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); 328 bkey_extent_entry_for_each(ptrs, entry) { 329 if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) { 330 struct gc_stripe *m = genradix_ptr(&c->gc_stripes, 331 entry->stripe_ptr.idx); 332 union bch_extent_entry *next_ptr; 333 334 bkey_extent_entry_for_each_from(ptrs, next_ptr, entry) 335 if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr) 336 goto found; 337 next_ptr = NULL; 338 found: 339 if (!next_ptr) { 340 bch_err(c, "aieee, found stripe ptr with no data ptr"); 341 continue; 342 } 343 344 if (!m || !m->alive || 345 !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block], 346 &next_ptr->ptr, 347 m->sectors)) { 348 bch2_bkey_extent_entry_drop(new, entry); 349 goto again; 350 } 351 } 352 } 353 } 354 355 if (0) { 356 printbuf_reset(&buf); 357 bch2_bkey_val_to_text(&buf, c, k); 358 bch_info(c, "updated %s", buf.buf); 359 360 printbuf_reset(&buf); 361 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(new)); 362 bch_info(c, "new key %s", buf.buf); 363 } 364 365 struct btree_iter iter; 366 bch2_trans_node_iter_init(trans, &iter, btree, new->k.p, 0, level, 367 BTREE_ITER_intent|BTREE_ITER_all_snapshots); 368 ret = bch2_btree_iter_traverse(&iter) ?: 369 bch2_trans_update(trans, &iter, new, 370 BTREE_UPDATE_internal_snapshot_node| 371 BTREE_TRIGGER_norun); 372 bch2_trans_iter_exit(trans, &iter); 373 if (ret) 374 goto err; 375 376 if (level) 377 bch2_btree_node_update_key_early(trans, btree, level - 1, k, new); 378 } 379 err: 380 printbuf_exit(&buf); 381 return ret; 382 } 383 384 int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca, 385 struct bkey_s_c k, 386 const struct bch_extent_ptr *ptr, 387 s64 sectors, enum bch_data_type ptr_data_type, 388 u8 b_gen, u8 bucket_data_type, 389 u32 *bucket_sectors) 390 { 391 struct bch_fs *c = trans->c; 392 size_t bucket_nr = PTR_BUCKET_NR(ca, ptr); 393 struct printbuf buf = PRINTBUF; 394 bool inserting = sectors > 0; 395 int ret = 0; 396 397 BUG_ON(!sectors); 398 399 if (gen_after(ptr->gen, b_gen)) { 400 bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_allocations); 401 log_fsck_err(trans, ptr_gen_newer_than_bucket_gen, 402 "bucket %u:%zu gen %u data type %s: ptr gen %u newer than bucket gen\n" 403 "while marking %s", 404 ptr->dev, bucket_nr, b_gen, 405 bch2_data_type_str(bucket_data_type ?: ptr_data_type), 406 ptr->gen, 407 (bch2_bkey_val_to_text(&buf, c, k), buf.buf)); 408 if (inserting) 409 goto err; 410 goto out; 411 } 412 413 if (gen_cmp(b_gen, ptr->gen) > BUCKET_GC_GEN_MAX) { 414 bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_allocations); 415 log_fsck_err(trans, ptr_too_stale, 416 "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n" 417 "while marking %s", 418 ptr->dev, bucket_nr, b_gen, 419 bch2_data_type_str(bucket_data_type ?: ptr_data_type), 420 ptr->gen, 421 (printbuf_reset(&buf), 422 bch2_bkey_val_to_text(&buf, c, k), buf.buf)); 423 if (inserting) 424 goto err; 425 goto out; 426 } 427 428 if (b_gen != ptr->gen && ptr->cached) { 429 ret = 1; 430 goto out; 431 } 432 433 if (b_gen != ptr->gen) { 434 bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_allocations); 435 log_fsck_err(trans, stale_dirty_ptr, 436 "bucket %u:%zu gen %u (mem gen %u) data type %s: stale dirty ptr (gen %u)\n" 437 "while marking %s", 438 ptr->dev, bucket_nr, b_gen, 439 bucket_gen_get(ca, bucket_nr), 440 bch2_data_type_str(bucket_data_type ?: ptr_data_type), 441 ptr->gen, 442 (printbuf_reset(&buf), 443 bch2_bkey_val_to_text(&buf, c, k), buf.buf)); 444 if (inserting) 445 goto err; 446 goto out; 447 } 448 449 if (bucket_data_type_mismatch(bucket_data_type, ptr_data_type)) { 450 bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_allocations); 451 log_fsck_err(trans, ptr_bucket_data_type_mismatch, 452 "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n" 453 "while marking %s", 454 ptr->dev, bucket_nr, b_gen, 455 bch2_data_type_str(bucket_data_type), 456 bch2_data_type_str(ptr_data_type), 457 (printbuf_reset(&buf), 458 bch2_bkey_val_to_text(&buf, c, k), buf.buf)); 459 if (inserting) 460 goto err; 461 goto out; 462 } 463 464 if ((u64) *bucket_sectors + sectors > U32_MAX) { 465 bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_allocations); 466 log_fsck_err(trans, bucket_sector_count_overflow, 467 "bucket %u:%zu gen %u data type %s sector count overflow: %u + %lli > U32_MAX\n" 468 "while marking %s", 469 ptr->dev, bucket_nr, b_gen, 470 bch2_data_type_str(bucket_data_type ?: ptr_data_type), 471 *bucket_sectors, sectors, 472 (printbuf_reset(&buf), 473 bch2_bkey_val_to_text(&buf, c, k), buf.buf)); 474 if (inserting) 475 goto err; 476 sectors = -*bucket_sectors; 477 } 478 479 *bucket_sectors += sectors; 480 out: 481 printbuf_exit(&buf); 482 return ret; 483 err: 484 fsck_err: 485 bch2_dump_trans_updates(trans); 486 bch2_inconsistent_error(c); 487 ret = -BCH_ERR_bucket_ref_update; 488 goto out; 489 } 490 491 void bch2_trans_account_disk_usage_change(struct btree_trans *trans) 492 { 493 struct bch_fs *c = trans->c; 494 u64 disk_res_sectors = trans->disk_res ? trans->disk_res->sectors : 0; 495 static int warned_disk_usage = 0; 496 bool warn = false; 497 498 percpu_down_read(&c->mark_lock); 499 struct bch_fs_usage_base *src = &trans->fs_usage_delta; 500 501 s64 added = src->btree + src->data + src->reserved; 502 503 /* 504 * Not allowed to reduce sectors_available except by getting a 505 * reservation: 506 */ 507 s64 should_not_have_added = added - (s64) disk_res_sectors; 508 if (unlikely(should_not_have_added > 0)) { 509 u64 old, new; 510 511 old = atomic64_read(&c->sectors_available); 512 do { 513 new = max_t(s64, 0, old - should_not_have_added); 514 } while (!atomic64_try_cmpxchg(&c->sectors_available, 515 &old, new)); 516 517 added -= should_not_have_added; 518 warn = true; 519 } 520 521 if (added > 0) { 522 trans->disk_res->sectors -= added; 523 this_cpu_sub(*c->online_reserved, added); 524 } 525 526 preempt_disable(); 527 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage); 528 acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64)); 529 preempt_enable(); 530 percpu_up_read(&c->mark_lock); 531 532 if (unlikely(warn) && !xchg(&warned_disk_usage, 1)) 533 bch2_trans_inconsistent(trans, 534 "disk usage increased %lli more than %llu sectors reserved)", 535 should_not_have_added, disk_res_sectors); 536 } 537 538 /* KEY_TYPE_extent: */ 539 540 static int __mark_pointer(struct btree_trans *trans, struct bch_dev *ca, 541 struct bkey_s_c k, 542 const struct extent_ptr_decoded *p, 543 s64 sectors, enum bch_data_type ptr_data_type, 544 struct bch_alloc_v4 *a, 545 bool insert) 546 { 547 u32 *dst_sectors = p->has_ec ? &a->stripe_sectors : 548 !p->ptr.cached ? &a->dirty_sectors : 549 &a->cached_sectors; 550 int ret = bch2_bucket_ref_update(trans, ca, k, &p->ptr, sectors, ptr_data_type, 551 a->gen, a->data_type, dst_sectors); 552 553 if (ret) 554 return ret; 555 if (insert) 556 alloc_data_type_set(a, ptr_data_type); 557 return 0; 558 } 559 560 static int bch2_trigger_pointer(struct btree_trans *trans, 561 enum btree_id btree_id, unsigned level, 562 struct bkey_s_c k, struct extent_ptr_decoded p, 563 const union bch_extent_entry *entry, 564 s64 *sectors, 565 enum btree_iter_update_trigger_flags flags) 566 { 567 struct bch_fs *c = trans->c; 568 bool insert = !(flags & BTREE_TRIGGER_overwrite); 569 struct printbuf buf = PRINTBUF; 570 int ret = 0; 571 572 struct bkey_i_backpointer bp; 573 bch2_extent_ptr_to_bp(c, btree_id, level, k, p, entry, &bp); 574 575 *sectors = insert ? bp.v.bucket_len : -(s64) bp.v.bucket_len; 576 577 struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev); 578 if (unlikely(!ca)) { 579 if (insert && p.ptr.dev != BCH_SB_MEMBER_INVALID) 580 ret = -BCH_ERR_trigger_pointer; 581 goto err; 582 } 583 584 struct bpos bucket = PTR_BUCKET_POS(ca, &p.ptr); 585 586 if (flags & BTREE_TRIGGER_transactional) { 587 struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket, 0); 588 ret = PTR_ERR_OR_ZERO(a) ?: 589 __mark_pointer(trans, ca, k, &p, *sectors, bp.v.data_type, &a->v, insert); 590 if (ret) 591 goto err; 592 593 if (!p.ptr.cached) { 594 ret = bch2_bucket_backpointer_mod(trans, k, &bp, insert); 595 if (ret) 596 goto err; 597 } 598 } 599 600 if (flags & BTREE_TRIGGER_gc) { 601 struct bucket *g = gc_bucket(ca, bucket.offset); 602 if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u\n %s", 603 p.ptr.dev, 604 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { 605 ret = -BCH_ERR_trigger_pointer; 606 goto err; 607 } 608 609 bucket_lock(g); 610 struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; 611 ret = __mark_pointer(trans, ca, k, &p, *sectors, bp.v.data_type, &new, insert); 612 alloc_to_bucket(g, new); 613 bucket_unlock(g); 614 615 if (!ret) 616 ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags); 617 } 618 err: 619 bch2_dev_put(ca); 620 printbuf_exit(&buf); 621 return ret; 622 } 623 624 static int bch2_trigger_stripe_ptr(struct btree_trans *trans, 625 struct bkey_s_c k, 626 struct extent_ptr_decoded p, 627 enum bch_data_type data_type, 628 s64 sectors, 629 enum btree_iter_update_trigger_flags flags) 630 { 631 if (flags & BTREE_TRIGGER_transactional) { 632 struct btree_iter iter; 633 struct bkey_i_stripe *s = bch2_bkey_get_mut_typed(trans, &iter, 634 BTREE_ID_stripes, POS(0, p.ec.idx), 635 BTREE_ITER_with_updates, stripe); 636 int ret = PTR_ERR_OR_ZERO(s); 637 if (unlikely(ret)) { 638 bch2_trans_inconsistent_on(bch2_err_matches(ret, ENOENT), trans, 639 "pointer to nonexistent stripe %llu", 640 (u64) p.ec.idx); 641 goto err; 642 } 643 644 if (!bch2_ptr_matches_stripe(&s->v, p)) { 645 bch2_trans_inconsistent(trans, 646 "stripe pointer doesn't match stripe %llu", 647 (u64) p.ec.idx); 648 ret = -BCH_ERR_trigger_stripe_pointer; 649 goto err; 650 } 651 652 stripe_blockcount_set(&s->v, p.ec.block, 653 stripe_blockcount_get(&s->v, p.ec.block) + 654 sectors); 655 656 struct disk_accounting_pos acc = { 657 .type = BCH_DISK_ACCOUNTING_replicas, 658 }; 659 bch2_bkey_to_replicas(&acc.replicas, bkey_i_to_s_c(&s->k_i)); 660 acc.replicas.data_type = data_type; 661 ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, false); 662 err: 663 bch2_trans_iter_exit(trans, &iter); 664 return ret; 665 } 666 667 if (flags & BTREE_TRIGGER_gc) { 668 struct bch_fs *c = trans->c; 669 670 struct gc_stripe *m = genradix_ptr_alloc(&c->gc_stripes, p.ec.idx, GFP_KERNEL); 671 if (!m) { 672 bch_err(c, "error allocating memory for gc_stripes, idx %llu", 673 (u64) p.ec.idx); 674 return -BCH_ERR_ENOMEM_mark_stripe_ptr; 675 } 676 677 mutex_lock(&c->ec_stripes_heap_lock); 678 679 if (!m || !m->alive) { 680 mutex_unlock(&c->ec_stripes_heap_lock); 681 struct printbuf buf = PRINTBUF; 682 bch2_bkey_val_to_text(&buf, c, k); 683 bch_err_ratelimited(c, "pointer to nonexistent stripe %llu\n while marking %s", 684 (u64) p.ec.idx, buf.buf); 685 printbuf_exit(&buf); 686 bch2_inconsistent_error(c); 687 return -BCH_ERR_trigger_stripe_pointer; 688 } 689 690 m->block_sectors[p.ec.block] += sectors; 691 692 struct disk_accounting_pos acc = { 693 .type = BCH_DISK_ACCOUNTING_replicas, 694 }; 695 memcpy(&acc.replicas, &m->r.e, replicas_entry_bytes(&m->r.e)); 696 mutex_unlock(&c->ec_stripes_heap_lock); 697 698 acc.replicas.data_type = data_type; 699 int ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, true); 700 if (ret) 701 return ret; 702 } 703 704 return 0; 705 } 706 707 static int __trigger_extent(struct btree_trans *trans, 708 enum btree_id btree_id, unsigned level, 709 struct bkey_s_c k, 710 enum btree_iter_update_trigger_flags flags, 711 s64 *replicas_sectors) 712 { 713 bool gc = flags & BTREE_TRIGGER_gc; 714 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); 715 const union bch_extent_entry *entry; 716 struct extent_ptr_decoded p; 717 enum bch_data_type data_type = bkey_is_btree_ptr(k.k) 718 ? BCH_DATA_btree 719 : BCH_DATA_user; 720 int ret = 0; 721 722 struct disk_accounting_pos acc_replicas_key = { 723 .type = BCH_DISK_ACCOUNTING_replicas, 724 .replicas.data_type = data_type, 725 .replicas.nr_devs = 0, 726 .replicas.nr_required = 1, 727 }; 728 729 struct disk_accounting_pos acct_compression_key = { 730 .type = BCH_DISK_ACCOUNTING_compression, 731 }; 732 u64 compression_acct[3] = { 1, 0, 0 }; 733 734 bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { 735 s64 disk_sectors = 0; 736 ret = bch2_trigger_pointer(trans, btree_id, level, k, p, entry, &disk_sectors, flags); 737 if (ret < 0) 738 return ret; 739 740 bool stale = ret > 0; 741 742 if (p.ptr.cached && stale) 743 continue; 744 745 if (p.ptr.cached) { 746 ret = bch2_mod_dev_cached_sectors(trans, p.ptr.dev, disk_sectors, gc); 747 if (ret) 748 return ret; 749 } else if (!p.has_ec) { 750 *replicas_sectors += disk_sectors; 751 replicas_entry_add_dev(&acc_replicas_key.replicas, p.ptr.dev); 752 } else { 753 ret = bch2_trigger_stripe_ptr(trans, k, p, data_type, disk_sectors, flags); 754 if (ret) 755 return ret; 756 757 /* 758 * There may be other dirty pointers in this extent, but 759 * if so they're not required for mounting if we have an 760 * erasure coded pointer in this extent: 761 */ 762 acc_replicas_key.replicas.nr_required = 0; 763 } 764 765 if (acct_compression_key.compression.type && 766 acct_compression_key.compression.type != p.crc.compression_type) { 767 if (flags & BTREE_TRIGGER_overwrite) 768 bch2_u64s_neg(compression_acct, ARRAY_SIZE(compression_acct)); 769 770 ret = bch2_disk_accounting_mod(trans, &acct_compression_key, compression_acct, 771 ARRAY_SIZE(compression_acct), gc); 772 if (ret) 773 return ret; 774 775 compression_acct[0] = 1; 776 compression_acct[1] = 0; 777 compression_acct[2] = 0; 778 } 779 780 acct_compression_key.compression.type = p.crc.compression_type; 781 if (p.crc.compression_type) { 782 compression_acct[1] += p.crc.uncompressed_size; 783 compression_acct[2] += p.crc.compressed_size; 784 } 785 } 786 787 if (acc_replicas_key.replicas.nr_devs) { 788 ret = bch2_disk_accounting_mod(trans, &acc_replicas_key, replicas_sectors, 1, gc); 789 if (ret) 790 return ret; 791 } 792 793 if (acc_replicas_key.replicas.nr_devs && !level && k.k->p.snapshot) { 794 struct disk_accounting_pos acc_snapshot_key = { 795 .type = BCH_DISK_ACCOUNTING_snapshot, 796 .snapshot.id = k.k->p.snapshot, 797 }; 798 ret = bch2_disk_accounting_mod(trans, &acc_snapshot_key, replicas_sectors, 1, gc); 799 if (ret) 800 return ret; 801 } 802 803 if (acct_compression_key.compression.type) { 804 if (flags & BTREE_TRIGGER_overwrite) 805 bch2_u64s_neg(compression_acct, ARRAY_SIZE(compression_acct)); 806 807 ret = bch2_disk_accounting_mod(trans, &acct_compression_key, compression_acct, 808 ARRAY_SIZE(compression_acct), gc); 809 if (ret) 810 return ret; 811 } 812 813 if (level) { 814 struct disk_accounting_pos acc_btree_key = { 815 .type = BCH_DISK_ACCOUNTING_btree, 816 .btree.id = btree_id, 817 }; 818 ret = bch2_disk_accounting_mod(trans, &acc_btree_key, replicas_sectors, 1, gc); 819 if (ret) 820 return ret; 821 } else { 822 bool insert = !(flags & BTREE_TRIGGER_overwrite); 823 struct disk_accounting_pos acc_inum_key = { 824 .type = BCH_DISK_ACCOUNTING_inum, 825 .inum.inum = k.k->p.inode, 826 }; 827 s64 v[3] = { 828 insert ? 1 : -1, 829 insert ? k.k->size : -((s64) k.k->size), 830 *replicas_sectors, 831 }; 832 ret = bch2_disk_accounting_mod(trans, &acc_inum_key, v, ARRAY_SIZE(v), gc); 833 if (ret) 834 return ret; 835 } 836 837 return 0; 838 } 839 840 int bch2_trigger_extent(struct btree_trans *trans, 841 enum btree_id btree, unsigned level, 842 struct bkey_s_c old, struct bkey_s new, 843 enum btree_iter_update_trigger_flags flags) 844 { 845 struct bch_fs *c = trans->c; 846 struct bkey_ptrs_c new_ptrs = bch2_bkey_ptrs_c(new.s_c); 847 struct bkey_ptrs_c old_ptrs = bch2_bkey_ptrs_c(old); 848 unsigned new_ptrs_bytes = (void *) new_ptrs.end - (void *) new_ptrs.start; 849 unsigned old_ptrs_bytes = (void *) old_ptrs.end - (void *) old_ptrs.start; 850 851 if (unlikely(flags & BTREE_TRIGGER_check_repair)) 852 return bch2_check_fix_ptrs(trans, btree, level, new.s_c, flags); 853 854 /* if pointers aren't changing - nothing to do: */ 855 if (new_ptrs_bytes == old_ptrs_bytes && 856 !memcmp(new_ptrs.start, 857 old_ptrs.start, 858 new_ptrs_bytes)) 859 return 0; 860 861 if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) { 862 s64 old_replicas_sectors = 0, new_replicas_sectors = 0; 863 864 if (old.k->type) { 865 int ret = __trigger_extent(trans, btree, level, old, 866 flags & ~BTREE_TRIGGER_insert, 867 &old_replicas_sectors); 868 if (ret) 869 return ret; 870 } 871 872 if (new.k->type) { 873 int ret = __trigger_extent(trans, btree, level, new.s_c, 874 flags & ~BTREE_TRIGGER_overwrite, 875 &new_replicas_sectors); 876 if (ret) 877 return ret; 878 } 879 880 int need_rebalance_delta = 0; 881 s64 need_rebalance_sectors_delta = 0; 882 883 s64 s = bch2_bkey_sectors_need_rebalance(c, old); 884 need_rebalance_delta -= s != 0; 885 need_rebalance_sectors_delta -= s; 886 887 s = bch2_bkey_sectors_need_rebalance(c, new.s_c); 888 need_rebalance_delta += s != 0; 889 need_rebalance_sectors_delta += s; 890 891 if ((flags & BTREE_TRIGGER_transactional) && need_rebalance_delta) { 892 int ret = bch2_btree_bit_mod_buffered(trans, BTREE_ID_rebalance_work, 893 new.k->p, need_rebalance_delta > 0); 894 if (ret) 895 return ret; 896 } 897 898 if (need_rebalance_sectors_delta) { 899 struct disk_accounting_pos acc = { 900 .type = BCH_DISK_ACCOUNTING_rebalance_work, 901 }; 902 int ret = bch2_disk_accounting_mod(trans, &acc, &need_rebalance_sectors_delta, 1, 903 flags & BTREE_TRIGGER_gc); 904 if (ret) 905 return ret; 906 } 907 } 908 909 return 0; 910 } 911 912 /* KEY_TYPE_reservation */ 913 914 static int __trigger_reservation(struct btree_trans *trans, 915 enum btree_id btree_id, unsigned level, struct bkey_s_c k, 916 enum btree_iter_update_trigger_flags flags) 917 { 918 if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) { 919 s64 sectors = k.k->size; 920 921 if (flags & BTREE_TRIGGER_overwrite) 922 sectors = -sectors; 923 924 struct disk_accounting_pos acc = { 925 .type = BCH_DISK_ACCOUNTING_persistent_reserved, 926 .persistent_reserved.nr_replicas = bkey_s_c_to_reservation(k).v->nr_replicas, 927 }; 928 929 return bch2_disk_accounting_mod(trans, &acc, §ors, 1, flags & BTREE_TRIGGER_gc); 930 } 931 932 return 0; 933 } 934 935 int bch2_trigger_reservation(struct btree_trans *trans, 936 enum btree_id btree_id, unsigned level, 937 struct bkey_s_c old, struct bkey_s new, 938 enum btree_iter_update_trigger_flags flags) 939 { 940 return trigger_run_overwrite_then_insert(__trigger_reservation, trans, btree_id, level, old, new, flags); 941 } 942 943 /* Mark superblocks: */ 944 945 static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans, 946 struct bch_dev *ca, u64 b, 947 enum bch_data_type type, 948 unsigned sectors) 949 { 950 struct bch_fs *c = trans->c; 951 struct btree_iter iter; 952 int ret = 0; 953 954 struct bkey_i_alloc_v4 *a = 955 bch2_trans_start_alloc_update_noupdate(trans, &iter, POS(ca->dev_idx, b)); 956 if (IS_ERR(a)) 957 return PTR_ERR(a); 958 959 if (a->v.data_type && type && a->v.data_type != type) { 960 bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_allocations); 961 log_fsck_err(trans, bucket_metadata_type_mismatch, 962 "bucket %llu:%llu gen %u different types of data in same bucket: %s, %s\n" 963 "while marking %s", 964 iter.pos.inode, iter.pos.offset, a->v.gen, 965 bch2_data_type_str(a->v.data_type), 966 bch2_data_type_str(type), 967 bch2_data_type_str(type)); 968 ret = -BCH_ERR_metadata_bucket_inconsistency; 969 goto err; 970 } 971 972 if (a->v.data_type != type || 973 a->v.dirty_sectors != sectors) { 974 a->v.data_type = type; 975 a->v.dirty_sectors = sectors; 976 ret = bch2_trans_update(trans, &iter, &a->k_i, 0); 977 } 978 err: 979 fsck_err: 980 bch2_trans_iter_exit(trans, &iter); 981 return ret; 982 } 983 984 static int bch2_mark_metadata_bucket(struct btree_trans *trans, struct bch_dev *ca, 985 u64 b, enum bch_data_type data_type, unsigned sectors, 986 enum btree_iter_update_trigger_flags flags) 987 { 988 struct bch_fs *c = trans->c; 989 int ret = 0; 990 991 struct bucket *g = gc_bucket(ca, b); 992 if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u when marking metadata type %s", 993 ca->dev_idx, bch2_data_type_str(data_type))) 994 goto err; 995 996 bucket_lock(g); 997 struct bch_alloc_v4 old = bucket_m_to_alloc(*g); 998 999 if (bch2_fs_inconsistent_on(g->data_type && 1000 g->data_type != data_type, c, 1001 "different types of data in same bucket: %s, %s", 1002 bch2_data_type_str(g->data_type), 1003 bch2_data_type_str(data_type))) 1004 goto err_unlock; 1005 1006 if (bch2_fs_inconsistent_on((u64) g->dirty_sectors + sectors > ca->mi.bucket_size, c, 1007 "bucket %u:%llu gen %u data type %s sector count overflow: %u + %u > bucket size", 1008 ca->dev_idx, b, g->gen, 1009 bch2_data_type_str(g->data_type ?: data_type), 1010 g->dirty_sectors, sectors)) 1011 goto err_unlock; 1012 1013 g->data_type = data_type; 1014 g->dirty_sectors += sectors; 1015 struct bch_alloc_v4 new = bucket_m_to_alloc(*g); 1016 bucket_unlock(g); 1017 ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags); 1018 return ret; 1019 err_unlock: 1020 bucket_unlock(g); 1021 err: 1022 return -BCH_ERR_metadata_bucket_inconsistency; 1023 } 1024 1025 int bch2_trans_mark_metadata_bucket(struct btree_trans *trans, 1026 struct bch_dev *ca, u64 b, 1027 enum bch_data_type type, unsigned sectors, 1028 enum btree_iter_update_trigger_flags flags) 1029 { 1030 BUG_ON(type != BCH_DATA_free && 1031 type != BCH_DATA_sb && 1032 type != BCH_DATA_journal); 1033 1034 /* 1035 * Backup superblock might be past the end of our normal usable space: 1036 */ 1037 if (b >= ca->mi.nbuckets) 1038 return 0; 1039 1040 if (flags & BTREE_TRIGGER_gc) 1041 return bch2_mark_metadata_bucket(trans, ca, b, type, sectors, flags); 1042 else if (flags & BTREE_TRIGGER_transactional) 1043 return commit_do(trans, NULL, NULL, 0, 1044 __bch2_trans_mark_metadata_bucket(trans, ca, b, type, sectors)); 1045 else 1046 BUG(); 1047 } 1048 1049 static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans, 1050 struct bch_dev *ca, u64 start, u64 end, 1051 enum bch_data_type type, u64 *bucket, unsigned *bucket_sectors, 1052 enum btree_iter_update_trigger_flags flags) 1053 { 1054 do { 1055 u64 b = sector_to_bucket(ca, start); 1056 unsigned sectors = 1057 min_t(u64, bucket_to_sector(ca, b + 1), end) - start; 1058 1059 if (b != *bucket && *bucket_sectors) { 1060 int ret = bch2_trans_mark_metadata_bucket(trans, ca, *bucket, 1061 type, *bucket_sectors, flags); 1062 if (ret) 1063 return ret; 1064 1065 *bucket_sectors = 0; 1066 } 1067 1068 *bucket = b; 1069 *bucket_sectors += sectors; 1070 start += sectors; 1071 } while (start < end); 1072 1073 return 0; 1074 } 1075 1076 static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, struct bch_dev *ca, 1077 enum btree_iter_update_trigger_flags flags) 1078 { 1079 struct bch_fs *c = trans->c; 1080 1081 mutex_lock(&c->sb_lock); 1082 struct bch_sb_layout layout = ca->disk_sb.sb->layout; 1083 mutex_unlock(&c->sb_lock); 1084 1085 u64 bucket = 0; 1086 unsigned i, bucket_sectors = 0; 1087 int ret; 1088 1089 for (i = 0; i < layout.nr_superblocks; i++) { 1090 u64 offset = le64_to_cpu(layout.sb_offset[i]); 1091 1092 if (offset == BCH_SB_SECTOR) { 1093 ret = bch2_trans_mark_metadata_sectors(trans, ca, 1094 0, BCH_SB_SECTOR, 1095 BCH_DATA_sb, &bucket, &bucket_sectors, flags); 1096 if (ret) 1097 return ret; 1098 } 1099 1100 ret = bch2_trans_mark_metadata_sectors(trans, ca, offset, 1101 offset + (1 << layout.sb_max_size_bits), 1102 BCH_DATA_sb, &bucket, &bucket_sectors, flags); 1103 if (ret) 1104 return ret; 1105 } 1106 1107 if (bucket_sectors) { 1108 ret = bch2_trans_mark_metadata_bucket(trans, ca, 1109 bucket, BCH_DATA_sb, bucket_sectors, flags); 1110 if (ret) 1111 return ret; 1112 } 1113 1114 for (i = 0; i < ca->journal.nr; i++) { 1115 ret = bch2_trans_mark_metadata_bucket(trans, ca, 1116 ca->journal.buckets[i], 1117 BCH_DATA_journal, ca->mi.bucket_size, flags); 1118 if (ret) 1119 return ret; 1120 } 1121 1122 return 0; 1123 } 1124 1125 int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca, 1126 enum btree_iter_update_trigger_flags flags) 1127 { 1128 int ret = bch2_trans_run(c, 1129 __bch2_trans_mark_dev_sb(trans, ca, flags)); 1130 bch_err_fn(c, ret); 1131 return ret; 1132 } 1133 1134 int bch2_trans_mark_dev_sbs_flags(struct bch_fs *c, 1135 enum btree_iter_update_trigger_flags flags) 1136 { 1137 for_each_online_member(c, ca) { 1138 int ret = bch2_trans_mark_dev_sb(c, ca, flags); 1139 if (ret) { 1140 percpu_ref_put(&ca->io_ref); 1141 return ret; 1142 } 1143 } 1144 1145 return 0; 1146 } 1147 1148 int bch2_trans_mark_dev_sbs(struct bch_fs *c) 1149 { 1150 return bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_transactional); 1151 } 1152 1153 bool bch2_is_superblock_bucket(struct bch_dev *ca, u64 b) 1154 { 1155 struct bch_sb_layout *layout = &ca->disk_sb.sb->layout; 1156 u64 b_offset = bucket_to_sector(ca, b); 1157 u64 b_end = bucket_to_sector(ca, b + 1); 1158 unsigned i; 1159 1160 if (!b) 1161 return true; 1162 1163 for (i = 0; i < layout->nr_superblocks; i++) { 1164 u64 offset = le64_to_cpu(layout->sb_offset[i]); 1165 u64 end = offset + (1 << layout->sb_max_size_bits); 1166 1167 if (!(offset >= b_end || end <= b_offset)) 1168 return true; 1169 } 1170 1171 for (i = 0; i < ca->journal.nr; i++) 1172 if (b == ca->journal.buckets[i]) 1173 return true; 1174 1175 return false; 1176 } 1177 1178 /* Disk reservations: */ 1179 1180 #define SECTORS_CACHE 1024 1181 1182 int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res, 1183 u64 sectors, enum bch_reservation_flags flags) 1184 { 1185 struct bch_fs_pcpu *pcpu; 1186 u64 old, get; 1187 u64 sectors_available; 1188 int ret; 1189 1190 percpu_down_read(&c->mark_lock); 1191 preempt_disable(); 1192 pcpu = this_cpu_ptr(c->pcpu); 1193 1194 if (sectors <= pcpu->sectors_available) 1195 goto out; 1196 1197 old = atomic64_read(&c->sectors_available); 1198 do { 1199 get = min((u64) sectors + SECTORS_CACHE, old); 1200 1201 if (get < sectors) { 1202 preempt_enable(); 1203 goto recalculate; 1204 } 1205 } while (!atomic64_try_cmpxchg(&c->sectors_available, 1206 &old, old - get)); 1207 1208 pcpu->sectors_available += get; 1209 1210 out: 1211 pcpu->sectors_available -= sectors; 1212 this_cpu_add(*c->online_reserved, sectors); 1213 res->sectors += sectors; 1214 1215 preempt_enable(); 1216 percpu_up_read(&c->mark_lock); 1217 return 0; 1218 1219 recalculate: 1220 mutex_lock(&c->sectors_available_lock); 1221 1222 percpu_u64_set(&c->pcpu->sectors_available, 0); 1223 sectors_available = avail_factor(__bch2_fs_usage_read_short(c).free); 1224 1225 if (sectors_available && (flags & BCH_DISK_RESERVATION_PARTIAL)) 1226 sectors = min(sectors, sectors_available); 1227 1228 if (sectors <= sectors_available || 1229 (flags & BCH_DISK_RESERVATION_NOFAIL)) { 1230 atomic64_set(&c->sectors_available, 1231 max_t(s64, 0, sectors_available - sectors)); 1232 this_cpu_add(*c->online_reserved, sectors); 1233 res->sectors += sectors; 1234 ret = 0; 1235 } else { 1236 atomic64_set(&c->sectors_available, sectors_available); 1237 ret = -BCH_ERR_ENOSPC_disk_reservation; 1238 } 1239 1240 mutex_unlock(&c->sectors_available_lock); 1241 percpu_up_read(&c->mark_lock); 1242 1243 return ret; 1244 } 1245 1246 /* Startup/shutdown: */ 1247 1248 void bch2_buckets_nouse_free(struct bch_fs *c) 1249 { 1250 for_each_member_device(c, ca) { 1251 kvfree_rcu_mightsleep(ca->buckets_nouse); 1252 ca->buckets_nouse = NULL; 1253 } 1254 } 1255 1256 int bch2_buckets_nouse_alloc(struct bch_fs *c) 1257 { 1258 for_each_member_device(c, ca) { 1259 BUG_ON(ca->buckets_nouse); 1260 1261 ca->buckets_nouse = bch2_kvmalloc(BITS_TO_LONGS(ca->mi.nbuckets) * 1262 sizeof(unsigned long), 1263 GFP_KERNEL|__GFP_ZERO); 1264 if (!ca->buckets_nouse) { 1265 bch2_dev_put(ca); 1266 return -BCH_ERR_ENOMEM_buckets_nouse; 1267 } 1268 } 1269 1270 return 0; 1271 } 1272 1273 static void bucket_gens_free_rcu(struct rcu_head *rcu) 1274 { 1275 struct bucket_gens *buckets = 1276 container_of(rcu, struct bucket_gens, rcu); 1277 1278 kvfree(buckets); 1279 } 1280 1281 int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) 1282 { 1283 struct bucket_gens *bucket_gens = NULL, *old_bucket_gens = NULL; 1284 bool resize = ca->bucket_gens != NULL; 1285 int ret; 1286 1287 if (resize) 1288 lockdep_assert_held(&c->state_lock); 1289 1290 if (resize && ca->buckets_nouse) 1291 return -BCH_ERR_no_resize_with_buckets_nouse; 1292 1293 bucket_gens = bch2_kvmalloc(struct_size(bucket_gens, b, nbuckets), 1294 GFP_KERNEL|__GFP_ZERO); 1295 if (!bucket_gens) { 1296 ret = -BCH_ERR_ENOMEM_bucket_gens; 1297 goto err; 1298 } 1299 1300 bucket_gens->first_bucket = ca->mi.first_bucket; 1301 bucket_gens->nbuckets = nbuckets; 1302 bucket_gens->nbuckets_minus_first = 1303 bucket_gens->nbuckets - bucket_gens->first_bucket; 1304 1305 old_bucket_gens = rcu_dereference_protected(ca->bucket_gens, 1); 1306 1307 if (resize) { 1308 bucket_gens->nbuckets = min(bucket_gens->nbuckets, 1309 old_bucket_gens->nbuckets); 1310 bucket_gens->nbuckets_minus_first = 1311 bucket_gens->nbuckets - bucket_gens->first_bucket; 1312 memcpy(bucket_gens->b, 1313 old_bucket_gens->b, 1314 bucket_gens->nbuckets); 1315 } 1316 1317 rcu_assign_pointer(ca->bucket_gens, bucket_gens); 1318 bucket_gens = old_bucket_gens; 1319 1320 nbuckets = ca->mi.nbuckets; 1321 1322 ret = 0; 1323 err: 1324 if (bucket_gens) 1325 call_rcu(&bucket_gens->rcu, bucket_gens_free_rcu); 1326 1327 return ret; 1328 } 1329 1330 void bch2_dev_buckets_free(struct bch_dev *ca) 1331 { 1332 kvfree(ca->buckets_nouse); 1333 kvfree(rcu_dereference_protected(ca->bucket_gens, 1)); 1334 free_percpu(ca->usage); 1335 } 1336 1337 int bch2_dev_buckets_alloc(struct bch_fs *c, struct bch_dev *ca) 1338 { 1339 ca->usage = alloc_percpu(struct bch_dev_usage); 1340 if (!ca->usage) 1341 return -BCH_ERR_ENOMEM_usage_init; 1342 1343 return bch2_dev_buckets_resize(c, ca, ca->mi.nbuckets); 1344 } 1345