1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "bcachefs.h" 4 #include "bcachefs_ioctl.h" 5 #include "btree_cache.h" 6 #include "btree_journal_iter.h" 7 #include "btree_update.h" 8 #include "btree_write_buffer.h" 9 #include "buckets.h" 10 #include "compress.h" 11 #include "disk_accounting.h" 12 #include "error.h" 13 #include "journal_io.h" 14 #include "replicas.h" 15 16 /* 17 * Notes on disk accounting: 18 * 19 * We have two parallel sets of counters to be concerned with, and both must be 20 * kept in sync. 21 * 22 * - Persistent/on disk accounting, stored in the accounting btree and updated 23 * via btree write buffer updates that treat new accounting keys as deltas to 24 * apply to existing values. But reading from a write buffer btree is 25 * expensive, so we also have 26 * 27 * - In memory accounting, where accounting is stored as an array of percpu 28 * counters, indexed by an eytzinger array of disk acounting keys/bpos (which 29 * are the same thing, excepting byte swabbing on big endian). 30 * 31 * Cheap to read, but non persistent. 32 * 33 * Disk accounting updates are generated by transactional triggers; these run as 34 * keys enter and leave the btree, and can compare old and new versions of keys; 35 * the output of these triggers are deltas to the various counters. 36 * 37 * Disk accounting updates are done as btree write buffer updates, where the 38 * counters in the disk accounting key are deltas that will be applied to the 39 * counter in the btree when the key is flushed by the write buffer (or journal 40 * replay). 41 * 42 * To do a disk accounting update: 43 * - initialize a disk_accounting_pos, to specify which counter is being update 44 * - initialize counter deltas, as an array of 1-3 s64s 45 * - call bch2_disk_accounting_mod() 46 * 47 * This queues up the accounting update to be done at transaction commit time. 48 * Underneath, it's a normal btree write buffer update. 49 * 50 * The transaction commit path is responsible for propagating updates to the in 51 * memory counters, with bch2_accounting_mem_mod(). 52 * 53 * The commit path also assigns every disk accounting update a unique version 54 * number, based on the journal sequence number and offset within that journal 55 * buffer; this is used by journal replay to determine which updates have been 56 * done. 57 * 58 * The transaction commit path also ensures that replicas entry accounting 59 * updates are properly marked in the superblock (so that we know whether we can 60 * mount without data being unavailable); it will update the superblock if 61 * bch2_accounting_mem_mod() tells it to. 62 */ 63 64 static const char * const disk_accounting_type_strs[] = { 65 #define x(t, n, ...) [n] = #t, 66 BCH_DISK_ACCOUNTING_TYPES() 67 #undef x 68 NULL 69 }; 70 71 static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos *pos, 72 s64 *d, unsigned nr) 73 { 74 struct bkey_i_accounting *acc = bkey_accounting_init(k); 75 76 acc->k.p = disk_accounting_pos_to_bpos(pos); 77 set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr); 78 79 memcpy_u64s_small(acc->v.d, d, nr); 80 } 81 82 static int bch2_accounting_update_sb_one(struct bch_fs *, struct bpos); 83 84 int bch2_disk_accounting_mod(struct btree_trans *trans, 85 struct disk_accounting_pos *k, 86 s64 *d, unsigned nr, bool gc) 87 { 88 /* Normalize: */ 89 switch (k->type) { 90 case BCH_DISK_ACCOUNTING_replicas: 91 bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp); 92 break; 93 } 94 95 BUG_ON(nr > BCH_ACCOUNTING_MAX_COUNTERS); 96 97 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i; 98 99 accounting_key_init(&k_i.k, k, d, nr); 100 101 if (unlikely(gc)) { 102 int ret = bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true); 103 if (ret == -BCH_ERR_btree_insert_need_mark_replicas) 104 ret = drop_locks_do(trans, 105 bch2_accounting_update_sb_one(trans->c, disk_accounting_pos_to_bpos(k))) ?: 106 bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true); 107 return ret; 108 } else { 109 return bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k); 110 } 111 } 112 113 int bch2_mod_dev_cached_sectors(struct btree_trans *trans, 114 unsigned dev, s64 sectors, 115 bool gc) 116 { 117 struct disk_accounting_pos acc = { 118 .type = BCH_DISK_ACCOUNTING_replicas, 119 }; 120 121 bch2_replicas_entry_cached(&acc.replicas, dev); 122 123 return bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc); 124 } 125 126 static inline bool is_zero(char *start, char *end) 127 { 128 BUG_ON(start > end); 129 130 for (; start < end; start++) 131 if (*start) 132 return false; 133 return true; 134 } 135 136 #define field_end(p, member) (((void *) (&p.member)) + sizeof(p.member)) 137 138 int bch2_accounting_validate(struct bch_fs *c, struct bkey_s_c k, 139 struct bkey_validate_context from) 140 { 141 struct disk_accounting_pos acc_k; 142 bpos_to_disk_accounting_pos(&acc_k, k.k->p); 143 void *end = &acc_k + 1; 144 int ret = 0; 145 146 bkey_fsck_err_on((from.flags & BCH_VALIDATE_commit) && 147 bversion_zero(k.k->bversion), 148 c, accounting_key_version_0, 149 "accounting key with version=0"); 150 151 switch (acc_k.type) { 152 case BCH_DISK_ACCOUNTING_nr_inodes: 153 end = field_end(acc_k, nr_inodes); 154 break; 155 case BCH_DISK_ACCOUNTING_persistent_reserved: 156 end = field_end(acc_k, persistent_reserved); 157 break; 158 case BCH_DISK_ACCOUNTING_replicas: 159 bkey_fsck_err_on(!acc_k.replicas.nr_devs, 160 c, accounting_key_replicas_nr_devs_0, 161 "accounting key replicas entry with nr_devs=0"); 162 163 bkey_fsck_err_on(acc_k.replicas.nr_required > acc_k.replicas.nr_devs || 164 (acc_k.replicas.nr_required > 1 && 165 acc_k.replicas.nr_required == acc_k.replicas.nr_devs), 166 c, accounting_key_replicas_nr_required_bad, 167 "accounting key replicas entry with bad nr_required"); 168 169 for (unsigned i = 0; i + 1 < acc_k.replicas.nr_devs; i++) 170 bkey_fsck_err_on(acc_k.replicas.devs[i] >= acc_k.replicas.devs[i + 1], 171 c, accounting_key_replicas_devs_unsorted, 172 "accounting key replicas entry with unsorted devs"); 173 174 end = (void *) &acc_k.replicas + replicas_entry_bytes(&acc_k.replicas); 175 break; 176 case BCH_DISK_ACCOUNTING_dev_data_type: 177 end = field_end(acc_k, dev_data_type); 178 break; 179 case BCH_DISK_ACCOUNTING_compression: 180 end = field_end(acc_k, compression); 181 break; 182 case BCH_DISK_ACCOUNTING_snapshot: 183 end = field_end(acc_k, snapshot); 184 break; 185 case BCH_DISK_ACCOUNTING_btree: 186 end = field_end(acc_k, btree); 187 break; 188 case BCH_DISK_ACCOUNTING_rebalance_work: 189 end = field_end(acc_k, rebalance_work); 190 break; 191 } 192 193 bkey_fsck_err_on(!is_zero(end, (void *) (&acc_k + 1)), 194 c, accounting_key_junk_at_end, 195 "junk at end of accounting key"); 196 fsck_err: 197 return ret; 198 } 199 200 void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k) 201 { 202 if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) { 203 prt_printf(out, "unknown type %u", k->type); 204 return; 205 } 206 207 prt_str(out, disk_accounting_type_strs[k->type]); 208 prt_str(out, " "); 209 210 switch (k->type) { 211 case BCH_DISK_ACCOUNTING_nr_inodes: 212 break; 213 case BCH_DISK_ACCOUNTING_persistent_reserved: 214 prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas); 215 break; 216 case BCH_DISK_ACCOUNTING_replicas: 217 bch2_replicas_entry_to_text(out, &k->replicas); 218 break; 219 case BCH_DISK_ACCOUNTING_dev_data_type: 220 prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev); 221 bch2_prt_data_type(out, k->dev_data_type.data_type); 222 break; 223 case BCH_DISK_ACCOUNTING_compression: 224 bch2_prt_compression_type(out, k->compression.type); 225 break; 226 case BCH_DISK_ACCOUNTING_snapshot: 227 prt_printf(out, "id=%u", k->snapshot.id); 228 break; 229 case BCH_DISK_ACCOUNTING_btree: 230 prt_str(out, "btree="); 231 bch2_btree_id_to_text(out, k->btree.id); 232 break; 233 } 234 } 235 236 void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) 237 { 238 struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k); 239 struct disk_accounting_pos acc_k; 240 bpos_to_disk_accounting_pos(&acc_k, k.k->p); 241 242 bch2_accounting_key_to_text(out, &acc_k); 243 244 for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++) 245 prt_printf(out, " %lli", acc.v->d[i]); 246 } 247 248 void bch2_accounting_swab(struct bkey_s k) 249 { 250 for (u64 *p = (u64 *) k.v; 251 p < (u64 *) bkey_val_end(k); 252 p++) 253 *p = swab64(*p); 254 } 255 256 static inline void __accounting_to_replicas(struct bch_replicas_entry_v1 *r, 257 struct disk_accounting_pos *acc) 258 { 259 unsafe_memcpy(r, &acc->replicas, 260 replicas_entry_bytes(&acc->replicas), 261 "variable length struct"); 262 } 263 264 static inline bool accounting_to_replicas(struct bch_replicas_entry_v1 *r, struct bpos p) 265 { 266 struct disk_accounting_pos acc_k; 267 bpos_to_disk_accounting_pos(&acc_k, p); 268 269 switch (acc_k.type) { 270 case BCH_DISK_ACCOUNTING_replicas: 271 __accounting_to_replicas(r, &acc_k); 272 return true; 273 default: 274 return false; 275 } 276 } 277 278 static int bch2_accounting_update_sb_one(struct bch_fs *c, struct bpos p) 279 { 280 struct bch_replicas_padded r; 281 return accounting_to_replicas(&r.e, p) 282 ? bch2_mark_replicas(c, &r.e) 283 : 0; 284 } 285 286 /* 287 * Ensure accounting keys being updated are present in the superblock, when 288 * applicable (i.e. replicas updates) 289 */ 290 int bch2_accounting_update_sb(struct btree_trans *trans) 291 { 292 for (struct jset_entry *i = trans->journal_entries; 293 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); 294 i = vstruct_next(i)) 295 if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) { 296 int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p); 297 if (ret) 298 return ret; 299 } 300 301 return 0; 302 } 303 304 static int __bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a) 305 { 306 struct bch_accounting_mem *acc = &c->accounting; 307 308 /* raced with another insert, already present: */ 309 if (eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 310 accounting_pos_cmp, &a.k->p) < acc->k.nr) 311 return 0; 312 313 struct accounting_mem_entry n = { 314 .pos = a.k->p, 315 .bversion = a.k->bversion, 316 .nr_counters = bch2_accounting_counters(a.k), 317 .v[0] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64), 318 sizeof(u64), GFP_KERNEL), 319 }; 320 321 if (!n.v[0]) 322 goto err; 323 324 if (acc->gc_running) { 325 n.v[1] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64), 326 sizeof(u64), GFP_KERNEL); 327 if (!n.v[1]) 328 goto err; 329 } 330 331 if (darray_push(&acc->k, n)) 332 goto err; 333 334 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 335 accounting_pos_cmp, NULL); 336 337 if (trace_accounting_mem_insert_enabled()) { 338 struct printbuf buf = PRINTBUF; 339 340 bch2_accounting_to_text(&buf, c, a.s_c); 341 trace_accounting_mem_insert(c, buf.buf); 342 printbuf_exit(&buf); 343 } 344 return 0; 345 err: 346 free_percpu(n.v[1]); 347 free_percpu(n.v[0]); 348 return -BCH_ERR_ENOMEM_disk_accounting; 349 } 350 351 int bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a, 352 enum bch_accounting_mode mode) 353 { 354 struct bch_replicas_padded r; 355 356 if (mode != BCH_ACCOUNTING_read && 357 accounting_to_replicas(&r.e, a.k->p) && 358 !bch2_replicas_marked_locked(c, &r.e)) 359 return -BCH_ERR_btree_insert_need_mark_replicas; 360 361 percpu_up_read(&c->mark_lock); 362 percpu_down_write(&c->mark_lock); 363 int ret = __bch2_accounting_mem_insert(c, a); 364 percpu_up_write(&c->mark_lock); 365 percpu_down_read(&c->mark_lock); 366 return ret; 367 } 368 369 static bool accounting_mem_entry_is_zero(struct accounting_mem_entry *e) 370 { 371 for (unsigned i = 0; i < e->nr_counters; i++) 372 if (percpu_u64_get(e->v[0] + i) || 373 (e->v[1] && 374 percpu_u64_get(e->v[1] + i))) 375 return false; 376 return true; 377 } 378 379 void bch2_accounting_mem_gc(struct bch_fs *c) 380 { 381 struct bch_accounting_mem *acc = &c->accounting; 382 383 percpu_down_write(&c->mark_lock); 384 struct accounting_mem_entry *dst = acc->k.data; 385 386 darray_for_each(acc->k, src) { 387 if (accounting_mem_entry_is_zero(src)) { 388 free_percpu(src->v[0]); 389 free_percpu(src->v[1]); 390 } else { 391 *dst++ = *src; 392 } 393 } 394 395 acc->k.nr = dst - acc->k.data; 396 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 397 accounting_pos_cmp, NULL); 398 percpu_up_write(&c->mark_lock); 399 } 400 401 /* 402 * Read out accounting keys for replicas entries, as an array of 403 * bch_replicas_usage entries. 404 * 405 * Note: this may be deprecated/removed at smoe point in the future and replaced 406 * with something more general, it exists to support the ioctl used by the 407 * 'bcachefs fs usage' command. 408 */ 409 int bch2_fs_replicas_usage_read(struct bch_fs *c, darray_char *usage) 410 { 411 struct bch_accounting_mem *acc = &c->accounting; 412 int ret = 0; 413 414 darray_init(usage); 415 416 percpu_down_read(&c->mark_lock); 417 darray_for_each(acc->k, i) { 418 struct { 419 struct bch_replicas_usage r; 420 u8 pad[BCH_BKEY_PTRS_MAX]; 421 } u; 422 423 if (!accounting_to_replicas(&u.r.r, i->pos)) 424 continue; 425 426 u64 sectors; 427 bch2_accounting_mem_read_counters(acc, i - acc->k.data, §ors, 1, false); 428 u.r.sectors = sectors; 429 430 ret = darray_make_room(usage, replicas_usage_bytes(&u.r)); 431 if (ret) 432 break; 433 434 memcpy(&darray_top(*usage), &u.r, replicas_usage_bytes(&u.r)); 435 usage->nr += replicas_usage_bytes(&u.r); 436 } 437 percpu_up_read(&c->mark_lock); 438 439 if (ret) 440 darray_exit(usage); 441 return ret; 442 } 443 444 int bch2_fs_accounting_read(struct bch_fs *c, darray_char *out_buf, unsigned accounting_types_mask) 445 { 446 447 struct bch_accounting_mem *acc = &c->accounting; 448 int ret = 0; 449 450 darray_init(out_buf); 451 452 percpu_down_read(&c->mark_lock); 453 darray_for_each(acc->k, i) { 454 struct disk_accounting_pos a_p; 455 bpos_to_disk_accounting_pos(&a_p, i->pos); 456 457 if (!(accounting_types_mask & BIT(a_p.type))) 458 continue; 459 460 ret = darray_make_room(out_buf, sizeof(struct bkey_i_accounting) + 461 sizeof(u64) * i->nr_counters); 462 if (ret) 463 break; 464 465 struct bkey_i_accounting *a_out = 466 bkey_accounting_init((void *) &darray_top(*out_buf)); 467 set_bkey_val_u64s(&a_out->k, i->nr_counters); 468 a_out->k.p = i->pos; 469 bch2_accounting_mem_read_counters(acc, i - acc->k.data, 470 a_out->v.d, i->nr_counters, false); 471 472 if (!bch2_accounting_key_is_zero(accounting_i_to_s_c(a_out))) 473 out_buf->nr += bkey_bytes(&a_out->k); 474 } 475 476 percpu_up_read(&c->mark_lock); 477 478 if (ret) 479 darray_exit(out_buf); 480 return ret; 481 } 482 483 static void bch2_accounting_free_counters(struct bch_accounting_mem *acc, bool gc) 484 { 485 darray_for_each(acc->k, e) { 486 free_percpu(e->v[gc]); 487 e->v[gc] = NULL; 488 } 489 } 490 491 int bch2_gc_accounting_start(struct bch_fs *c) 492 { 493 struct bch_accounting_mem *acc = &c->accounting; 494 int ret = 0; 495 496 percpu_down_write(&c->mark_lock); 497 darray_for_each(acc->k, e) { 498 e->v[1] = __alloc_percpu_gfp(e->nr_counters * sizeof(u64), 499 sizeof(u64), GFP_KERNEL); 500 if (!e->v[1]) { 501 bch2_accounting_free_counters(acc, true); 502 ret = -BCH_ERR_ENOMEM_disk_accounting; 503 break; 504 } 505 } 506 507 acc->gc_running = !ret; 508 percpu_up_write(&c->mark_lock); 509 510 return ret; 511 } 512 513 int bch2_gc_accounting_done(struct bch_fs *c) 514 { 515 struct bch_accounting_mem *acc = &c->accounting; 516 struct btree_trans *trans = bch2_trans_get(c); 517 struct printbuf buf = PRINTBUF; 518 struct bpos pos = POS_MIN; 519 int ret = 0; 520 521 percpu_down_write(&c->mark_lock); 522 while (1) { 523 unsigned idx = eytzinger0_find_ge(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 524 accounting_pos_cmp, &pos); 525 526 if (idx >= acc->k.nr) 527 break; 528 529 struct accounting_mem_entry *e = acc->k.data + idx; 530 pos = bpos_successor(e->pos); 531 532 struct disk_accounting_pos acc_k; 533 bpos_to_disk_accounting_pos(&acc_k, e->pos); 534 535 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR) 536 continue; 537 538 u64 src_v[BCH_ACCOUNTING_MAX_COUNTERS]; 539 u64 dst_v[BCH_ACCOUNTING_MAX_COUNTERS]; 540 541 unsigned nr = e->nr_counters; 542 bch2_accounting_mem_read_counters(acc, idx, dst_v, nr, false); 543 bch2_accounting_mem_read_counters(acc, idx, src_v, nr, true); 544 545 if (memcmp(dst_v, src_v, nr * sizeof(u64))) { 546 printbuf_reset(&buf); 547 prt_str(&buf, "accounting mismatch for "); 548 bch2_accounting_key_to_text(&buf, &acc_k); 549 550 prt_str(&buf, ": got"); 551 for (unsigned j = 0; j < nr; j++) 552 prt_printf(&buf, " %llu", dst_v[j]); 553 554 prt_str(&buf, " should be"); 555 for (unsigned j = 0; j < nr; j++) 556 prt_printf(&buf, " %llu", src_v[j]); 557 558 for (unsigned j = 0; j < nr; j++) 559 src_v[j] -= dst_v[j]; 560 561 if (fsck_err(trans, accounting_mismatch, "%s", buf.buf)) { 562 percpu_up_write(&c->mark_lock); 563 ret = commit_do(trans, NULL, NULL, 0, 564 bch2_disk_accounting_mod(trans, &acc_k, src_v, nr, false)); 565 percpu_down_write(&c->mark_lock); 566 if (ret) 567 goto err; 568 569 if (!test_bit(BCH_FS_may_go_rw, &c->flags)) { 570 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); 571 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i; 572 573 accounting_key_init(&k_i.k, &acc_k, src_v, nr); 574 bch2_accounting_mem_mod_locked(trans, 575 bkey_i_to_s_c_accounting(&k_i.k), 576 BCH_ACCOUNTING_normal); 577 578 preempt_disable(); 579 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage); 580 struct bch_fs_usage_base *src = &trans->fs_usage_delta; 581 acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64)); 582 preempt_enable(); 583 } 584 } 585 } 586 } 587 err: 588 fsck_err: 589 percpu_up_write(&c->mark_lock); 590 printbuf_exit(&buf); 591 bch2_trans_put(trans); 592 bch_err_fn(c, ret); 593 return ret; 594 } 595 596 static int accounting_read_key(struct btree_trans *trans, struct bkey_s_c k) 597 { 598 struct bch_fs *c = trans->c; 599 600 if (k.k->type != KEY_TYPE_accounting) 601 return 0; 602 603 percpu_down_read(&c->mark_lock); 604 int ret = bch2_accounting_mem_mod_locked(trans, bkey_s_c_to_accounting(k), 605 BCH_ACCOUNTING_read); 606 percpu_up_read(&c->mark_lock); 607 return ret; 608 } 609 610 static int bch2_disk_accounting_validate_late(struct btree_trans *trans, 611 struct disk_accounting_pos acc, 612 u64 *v, unsigned nr) 613 { 614 struct bch_fs *c = trans->c; 615 struct printbuf buf = PRINTBUF; 616 int ret = 0, invalid_dev = -1; 617 618 switch (acc.type) { 619 case BCH_DISK_ACCOUNTING_replicas: { 620 struct bch_replicas_padded r; 621 __accounting_to_replicas(&r.e, &acc); 622 623 for (unsigned i = 0; i < r.e.nr_devs; i++) 624 if (r.e.devs[i] != BCH_SB_MEMBER_INVALID && 625 !bch2_dev_exists(c, r.e.devs[i])) { 626 invalid_dev = r.e.devs[i]; 627 goto invalid_device; 628 } 629 630 /* 631 * All replicas entry checks except for invalid device are done 632 * in bch2_accounting_validate 633 */ 634 BUG_ON(bch2_replicas_entry_validate(&r.e, c, &buf)); 635 636 if (fsck_err_on(!bch2_replicas_marked_locked(c, &r.e), 637 trans, accounting_replicas_not_marked, 638 "accounting not marked in superblock replicas\n %s", 639 (printbuf_reset(&buf), 640 bch2_accounting_key_to_text(&buf, &acc), 641 buf.buf))) { 642 /* 643 * We're not RW yet and still single threaded, dropping 644 * and retaking lock is ok: 645 */ 646 percpu_up_write(&c->mark_lock); 647 ret = bch2_mark_replicas(c, &r.e); 648 if (ret) 649 goto fsck_err; 650 percpu_down_write(&c->mark_lock); 651 } 652 break; 653 } 654 655 case BCH_DISK_ACCOUNTING_dev_data_type: 656 if (!bch2_dev_exists(c, acc.dev_data_type.dev)) { 657 invalid_dev = acc.dev_data_type.dev; 658 goto invalid_device; 659 } 660 break; 661 } 662 663 fsck_err: 664 printbuf_exit(&buf); 665 return ret; 666 invalid_device: 667 if (fsck_err(trans, accounting_to_invalid_device, 668 "accounting entry points to invalid device %i\n %s", 669 invalid_dev, 670 (printbuf_reset(&buf), 671 bch2_accounting_key_to_text(&buf, &acc), 672 buf.buf))) { 673 for (unsigned i = 0; i < nr; i++) 674 v[i] = -v[i]; 675 676 ret = commit_do(trans, NULL, NULL, 0, 677 bch2_disk_accounting_mod(trans, &acc, v, nr, false)) ?: 678 -BCH_ERR_remove_disk_accounting_entry; 679 } else { 680 ret = -BCH_ERR_remove_disk_accounting_entry; 681 } 682 goto fsck_err; 683 } 684 685 /* 686 * At startup time, initialize the in memory accounting from the btree (and 687 * journal) 688 */ 689 int bch2_accounting_read(struct bch_fs *c) 690 { 691 struct bch_accounting_mem *acc = &c->accounting; 692 struct btree_trans *trans = bch2_trans_get(c); 693 struct printbuf buf = PRINTBUF; 694 695 /* 696 * We might run more than once if we rewind to start topology repair or 697 * btree node scan - and those might cause us to get different results, 698 * so we can't just skip if we've already run. 699 * 700 * Instead, zero out any accounting we have: 701 */ 702 percpu_down_write(&c->mark_lock); 703 darray_for_each(acc->k, e) 704 percpu_memset(e->v[0], 0, sizeof(u64) * e->nr_counters); 705 for_each_member_device(c, ca) 706 percpu_memset(ca->usage, 0, sizeof(*ca->usage)); 707 percpu_memset(c->usage, 0, sizeof(*c->usage)); 708 percpu_up_write(&c->mark_lock); 709 710 struct btree_iter iter; 711 bch2_trans_iter_init(trans, &iter, BTREE_ID_accounting, POS_MIN, 712 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots); 713 iter.flags &= ~BTREE_ITER_with_journal; 714 int ret = for_each_btree_key_continue(trans, iter, 715 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({ 716 struct bkey u; 717 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u); 718 719 if (k.k->type != KEY_TYPE_accounting) 720 continue; 721 722 struct disk_accounting_pos acc_k; 723 bpos_to_disk_accounting_pos(&acc_k, k.k->p); 724 725 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR) 726 break; 727 728 if (!bch2_accounting_is_mem(acc_k)) { 729 struct disk_accounting_pos next = { .type = acc_k.type + 1 }; 730 bch2_btree_iter_set_pos(&iter, disk_accounting_pos_to_bpos(&next)); 731 continue; 732 } 733 734 accounting_read_key(trans, k); 735 })); 736 if (ret) 737 goto err; 738 739 struct journal_keys *keys = &c->journal_keys; 740 struct journal_key *dst = keys->data; 741 move_gap(keys, keys->nr); 742 743 darray_for_each(*keys, i) { 744 if (i->k->k.type == KEY_TYPE_accounting) { 745 struct disk_accounting_pos acc_k; 746 bpos_to_disk_accounting_pos(&acc_k, i->k->k.p); 747 748 if (!bch2_accounting_is_mem(acc_k)) 749 continue; 750 751 struct bkey_s_c k = bkey_i_to_s_c(i->k); 752 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, 753 sizeof(acc->k.data[0]), 754 accounting_pos_cmp, &k.k->p); 755 756 bool applied = idx < acc->k.nr && 757 bversion_cmp(acc->k.data[idx].bversion, k.k->bversion) >= 0; 758 759 if (applied) 760 continue; 761 762 if (i + 1 < &darray_top(*keys) && 763 i[1].k->k.type == KEY_TYPE_accounting && 764 !journal_key_cmp(i, i + 1)) { 765 WARN_ON(bversion_cmp(i[0].k->k.bversion, i[1].k->k.bversion) >= 0); 766 767 i[1].journal_seq = i[0].journal_seq; 768 769 bch2_accounting_accumulate(bkey_i_to_accounting(i[1].k), 770 bkey_s_c_to_accounting(k)); 771 continue; 772 } 773 774 ret = accounting_read_key(trans, k); 775 if (ret) 776 goto err; 777 } 778 779 *dst++ = *i; 780 } 781 keys->gap = keys->nr = dst - keys->data; 782 783 percpu_down_write(&c->mark_lock); 784 785 darray_for_each_reverse(acc->k, i) { 786 struct disk_accounting_pos acc_k; 787 bpos_to_disk_accounting_pos(&acc_k, i->pos); 788 789 u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; 790 memset(v, 0, sizeof(v)); 791 792 for (unsigned j = 0; j < i->nr_counters; j++) 793 v[j] = percpu_u64_get(i->v[0] + j); 794 795 /* 796 * If the entry counters are zeroed, it should be treated as 797 * nonexistent - it might point to an invalid device. 798 * 799 * Remove it, so that if it's re-added it gets re-marked in the 800 * superblock: 801 */ 802 ret = bch2_is_zero(v, sizeof(v[0]) * i->nr_counters) 803 ? -BCH_ERR_remove_disk_accounting_entry 804 : bch2_disk_accounting_validate_late(trans, acc_k, v, i->nr_counters); 805 806 if (ret == -BCH_ERR_remove_disk_accounting_entry) { 807 free_percpu(i->v[0]); 808 free_percpu(i->v[1]); 809 darray_remove_item(&acc->k, i); 810 ret = 0; 811 continue; 812 } 813 814 if (ret) 815 goto fsck_err; 816 } 817 818 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 819 accounting_pos_cmp, NULL); 820 821 preempt_disable(); 822 struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage); 823 824 for (unsigned i = 0; i < acc->k.nr; i++) { 825 struct disk_accounting_pos k; 826 bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos); 827 828 u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; 829 bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false); 830 831 switch (k.type) { 832 case BCH_DISK_ACCOUNTING_persistent_reserved: 833 usage->reserved += v[0] * k.persistent_reserved.nr_replicas; 834 break; 835 case BCH_DISK_ACCOUNTING_replicas: 836 fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]); 837 break; 838 case BCH_DISK_ACCOUNTING_dev_data_type: 839 rcu_read_lock(); 840 struct bch_dev *ca = bch2_dev_rcu_noerror(c, k.dev_data_type.dev); 841 if (ca) { 842 struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type]; 843 percpu_u64_set(&d->buckets, v[0]); 844 percpu_u64_set(&d->sectors, v[1]); 845 percpu_u64_set(&d->fragmented, v[2]); 846 847 if (k.dev_data_type.data_type == BCH_DATA_sb || 848 k.dev_data_type.data_type == BCH_DATA_journal) 849 usage->hidden += v[0] * ca->mi.bucket_size; 850 } 851 rcu_read_unlock(); 852 break; 853 } 854 } 855 preempt_enable(); 856 fsck_err: 857 percpu_up_write(&c->mark_lock); 858 err: 859 printbuf_exit(&buf); 860 bch2_trans_put(trans); 861 bch_err_fn(c, ret); 862 return ret; 863 } 864 865 int bch2_dev_usage_remove(struct bch_fs *c, unsigned dev) 866 { 867 return bch2_trans_run(c, 868 bch2_btree_write_buffer_flush_sync(trans) ?: 869 for_each_btree_key_commit(trans, iter, BTREE_ID_accounting, POS_MIN, 870 BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({ 871 struct disk_accounting_pos acc; 872 bpos_to_disk_accounting_pos(&acc, k.k->p); 873 874 acc.type == BCH_DISK_ACCOUNTING_dev_data_type && 875 acc.dev_data_type.dev == dev 876 ? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0) 877 : 0; 878 })) ?: 879 bch2_btree_write_buffer_flush_sync(trans)); 880 } 881 882 int bch2_dev_usage_init(struct bch_dev *ca, bool gc) 883 { 884 struct bch_fs *c = ca->fs; 885 struct disk_accounting_pos acc = { 886 .type = BCH_DISK_ACCOUNTING_dev_data_type, 887 .dev_data_type.dev = ca->dev_idx, 888 .dev_data_type.data_type = BCH_DATA_free, 889 }; 890 u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 }; 891 892 int ret = bch2_trans_do(c, ({ 893 bch2_disk_accounting_mod(trans, &acc, v, ARRAY_SIZE(v), gc) ?: 894 (!gc ? bch2_trans_commit(trans, NULL, NULL, 0) : 0); 895 })); 896 bch_err_fn(c, ret); 897 return ret; 898 } 899 900 void bch2_verify_accounting_clean(struct bch_fs *c) 901 { 902 bool mismatch = false; 903 struct bch_fs_usage_base base = {}, base_inmem = {}; 904 905 bch2_trans_run(c, 906 for_each_btree_key(trans, iter, 907 BTREE_ID_accounting, POS_MIN, 908 BTREE_ITER_all_snapshots, k, ({ 909 u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; 910 struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k); 911 unsigned nr = bch2_accounting_counters(k.k); 912 913 struct disk_accounting_pos acc_k; 914 bpos_to_disk_accounting_pos(&acc_k, k.k->p); 915 916 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR) 917 break; 918 919 if (!bch2_accounting_is_mem(acc_k)) { 920 struct disk_accounting_pos next = { .type = acc_k.type + 1 }; 921 bch2_btree_iter_set_pos(&iter, disk_accounting_pos_to_bpos(&next)); 922 continue; 923 } 924 925 bch2_accounting_mem_read(c, k.k->p, v, nr); 926 927 if (memcmp(a.v->d, v, nr * sizeof(u64))) { 928 struct printbuf buf = PRINTBUF; 929 930 bch2_bkey_val_to_text(&buf, c, k); 931 prt_str(&buf, " !="); 932 for (unsigned j = 0; j < nr; j++) 933 prt_printf(&buf, " %llu", v[j]); 934 935 pr_err("%s", buf.buf); 936 printbuf_exit(&buf); 937 mismatch = true; 938 } 939 940 switch (acc_k.type) { 941 case BCH_DISK_ACCOUNTING_persistent_reserved: 942 base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0]; 943 break; 944 case BCH_DISK_ACCOUNTING_replicas: 945 fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]); 946 break; 947 case BCH_DISK_ACCOUNTING_dev_data_type: { 948 rcu_read_lock(); 949 struct bch_dev *ca = bch2_dev_rcu_noerror(c, acc_k.dev_data_type.dev); 950 if (!ca) { 951 rcu_read_unlock(); 952 continue; 953 } 954 955 v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets); 956 v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors); 957 v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented); 958 rcu_read_unlock(); 959 960 if (memcmp(a.v->d, v, 3 * sizeof(u64))) { 961 struct printbuf buf = PRINTBUF; 962 963 bch2_bkey_val_to_text(&buf, c, k); 964 prt_str(&buf, " in mem"); 965 for (unsigned j = 0; j < nr; j++) 966 prt_printf(&buf, " %llu", v[j]); 967 968 pr_err("dev accounting mismatch: %s", buf.buf); 969 printbuf_exit(&buf); 970 mismatch = true; 971 } 972 } 973 } 974 975 0; 976 }))); 977 978 acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64)); 979 980 #define check(x) \ 981 if (base.x != base_inmem.x) { \ 982 pr_err("fs_usage_base.%s mismatch: %llu != %llu", #x, base.x, base_inmem.x); \ 983 mismatch = true; \ 984 } 985 986 //check(hidden); 987 check(btree); 988 check(data); 989 check(cached); 990 check(reserved); 991 check(nr_inodes); 992 993 WARN_ON(mismatch); 994 } 995 996 void bch2_accounting_gc_free(struct bch_fs *c) 997 { 998 lockdep_assert_held(&c->mark_lock); 999 1000 struct bch_accounting_mem *acc = &c->accounting; 1001 1002 bch2_accounting_free_counters(acc, true); 1003 acc->gc_running = false; 1004 } 1005 1006 void bch2_fs_accounting_exit(struct bch_fs *c) 1007 { 1008 struct bch_accounting_mem *acc = &c->accounting; 1009 1010 bch2_accounting_free_counters(acc, false); 1011 darray_exit(&acc->k); 1012 } 1013