1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "bcachefs.h" 4 #include "bcachefs_ioctl.h" 5 #include "btree_cache.h" 6 #include "btree_journal_iter.h" 7 #include "btree_update.h" 8 #include "btree_write_buffer.h" 9 #include "buckets.h" 10 #include "compress.h" 11 #include "disk_accounting.h" 12 #include "error.h" 13 #include "journal_io.h" 14 #include "replicas.h" 15 16 /* 17 * Notes on disk accounting: 18 * 19 * We have two parallel sets of counters to be concerned with, and both must be 20 * kept in sync. 21 * 22 * - Persistent/on disk accounting, stored in the accounting btree and updated 23 * via btree write buffer updates that treat new accounting keys as deltas to 24 * apply to existing values. But reading from a write buffer btree is 25 * expensive, so we also have 26 * 27 * - In memory accounting, where accounting is stored as an array of percpu 28 * counters, indexed by an eytzinger array of disk acounting keys/bpos (which 29 * are the same thing, excepting byte swabbing on big endian). 30 * 31 * Cheap to read, but non persistent. 32 * 33 * Disk accounting updates are generated by transactional triggers; these run as 34 * keys enter and leave the btree, and can compare old and new versions of keys; 35 * the output of these triggers are deltas to the various counters. 36 * 37 * Disk accounting updates are done as btree write buffer updates, where the 38 * counters in the disk accounting key are deltas that will be applied to the 39 * counter in the btree when the key is flushed by the write buffer (or journal 40 * replay). 41 * 42 * To do a disk accounting update: 43 * - initialize a disk_accounting_pos, to specify which counter is being update 44 * - initialize counter deltas, as an array of 1-3 s64s 45 * - call bch2_disk_accounting_mod() 46 * 47 * This queues up the accounting update to be done at transaction commit time. 48 * Underneath, it's a normal btree write buffer update. 49 * 50 * The transaction commit path is responsible for propagating updates to the in 51 * memory counters, with bch2_accounting_mem_mod(). 52 * 53 * The commit path also assigns every disk accounting update a unique version 54 * number, based on the journal sequence number and offset within that journal 55 * buffer; this is used by journal replay to determine which updates have been 56 * done. 57 * 58 * The transaction commit path also ensures that replicas entry accounting 59 * updates are properly marked in the superblock (so that we know whether we can 60 * mount without data being unavailable); it will update the superblock if 61 * bch2_accounting_mem_mod() tells it to. 62 */ 63 64 static const char * const disk_accounting_type_strs[] = { 65 #define x(t, n, ...) [n] = #t, 66 BCH_DISK_ACCOUNTING_TYPES() 67 #undef x 68 NULL 69 }; 70 71 static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos *pos, 72 s64 *d, unsigned nr) 73 { 74 struct bkey_i_accounting *acc = bkey_accounting_init(k); 75 76 acc->k.p = disk_accounting_pos_to_bpos(pos); 77 set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr); 78 79 memcpy_u64s_small(acc->v.d, d, nr); 80 } 81 82 static int bch2_accounting_update_sb_one(struct bch_fs *, struct bpos); 83 84 int bch2_disk_accounting_mod(struct btree_trans *trans, 85 struct disk_accounting_pos *k, 86 s64 *d, unsigned nr, bool gc) 87 { 88 /* Normalize: */ 89 switch (k->type) { 90 case BCH_DISK_ACCOUNTING_replicas: 91 bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp); 92 break; 93 } 94 95 BUG_ON(nr > BCH_ACCOUNTING_MAX_COUNTERS); 96 97 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i; 98 99 accounting_key_init(&k_i.k, k, d, nr); 100 101 if (unlikely(gc)) { 102 int ret = bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true); 103 if (ret == -BCH_ERR_btree_insert_need_mark_replicas) 104 ret = drop_locks_do(trans, 105 bch2_accounting_update_sb_one(trans->c, disk_accounting_pos_to_bpos(k))) ?: 106 bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true); 107 return ret; 108 } else { 109 return bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k); 110 } 111 } 112 113 int bch2_mod_dev_cached_sectors(struct btree_trans *trans, 114 unsigned dev, s64 sectors, 115 bool gc) 116 { 117 struct disk_accounting_pos acc; 118 memset(&acc, 0, sizeof(acc)); 119 acc.type = BCH_DISK_ACCOUNTING_replicas; 120 bch2_replicas_entry_cached(&acc.replicas, dev); 121 122 return bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc); 123 } 124 125 static inline bool is_zero(char *start, char *end) 126 { 127 BUG_ON(start > end); 128 129 for (; start < end; start++) 130 if (*start) 131 return false; 132 return true; 133 } 134 135 #define field_end(p, member) (((void *) (&p.member)) + sizeof(p.member)) 136 137 static const unsigned bch2_accounting_type_nr_counters[] = { 138 #define x(f, id, nr) [BCH_DISK_ACCOUNTING_##f] = nr, 139 BCH_DISK_ACCOUNTING_TYPES() 140 #undef x 141 }; 142 143 int bch2_accounting_validate(struct bch_fs *c, struct bkey_s_c k, 144 struct bkey_validate_context from) 145 { 146 struct disk_accounting_pos acc_k; 147 bpos_to_disk_accounting_pos(&acc_k, k.k->p); 148 void *end = &acc_k + 1; 149 int ret = 0; 150 151 bkey_fsck_err_on((from.flags & BCH_VALIDATE_commit) && 152 bversion_zero(k.k->bversion), 153 c, accounting_key_version_0, 154 "accounting key with version=0"); 155 156 switch (acc_k.type) { 157 case BCH_DISK_ACCOUNTING_nr_inodes: 158 end = field_end(acc_k, nr_inodes); 159 break; 160 case BCH_DISK_ACCOUNTING_persistent_reserved: 161 end = field_end(acc_k, persistent_reserved); 162 break; 163 case BCH_DISK_ACCOUNTING_replicas: 164 bkey_fsck_err_on(!acc_k.replicas.nr_devs, 165 c, accounting_key_replicas_nr_devs_0, 166 "accounting key replicas entry with nr_devs=0"); 167 168 bkey_fsck_err_on(acc_k.replicas.nr_required > acc_k.replicas.nr_devs || 169 (acc_k.replicas.nr_required > 1 && 170 acc_k.replicas.nr_required == acc_k.replicas.nr_devs), 171 c, accounting_key_replicas_nr_required_bad, 172 "accounting key replicas entry with bad nr_required"); 173 174 for (unsigned i = 0; i + 1 < acc_k.replicas.nr_devs; i++) 175 bkey_fsck_err_on(acc_k.replicas.devs[i] >= acc_k.replicas.devs[i + 1], 176 c, accounting_key_replicas_devs_unsorted, 177 "accounting key replicas entry with unsorted devs"); 178 179 end = (void *) &acc_k.replicas + replicas_entry_bytes(&acc_k.replicas); 180 break; 181 case BCH_DISK_ACCOUNTING_dev_data_type: 182 end = field_end(acc_k, dev_data_type); 183 break; 184 case BCH_DISK_ACCOUNTING_compression: 185 end = field_end(acc_k, compression); 186 break; 187 case BCH_DISK_ACCOUNTING_snapshot: 188 end = field_end(acc_k, snapshot); 189 break; 190 case BCH_DISK_ACCOUNTING_btree: 191 end = field_end(acc_k, btree); 192 break; 193 case BCH_DISK_ACCOUNTING_rebalance_work: 194 end = field_end(acc_k, rebalance_work); 195 break; 196 } 197 198 bkey_fsck_err_on(!is_zero(end, (void *) (&acc_k + 1)), 199 c, accounting_key_junk_at_end, 200 "junk at end of accounting key"); 201 202 bkey_fsck_err_on(bch2_accounting_counters(k.k) != bch2_accounting_type_nr_counters[acc_k.type], 203 c, accounting_key_nr_counters_wrong, 204 "accounting key with %u counters, should be %u", 205 bch2_accounting_counters(k.k), bch2_accounting_type_nr_counters[acc_k.type]); 206 fsck_err: 207 return ret; 208 } 209 210 void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k) 211 { 212 if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) { 213 prt_printf(out, "unknown type %u", k->type); 214 return; 215 } 216 217 prt_str(out, disk_accounting_type_strs[k->type]); 218 prt_str(out, " "); 219 220 switch (k->type) { 221 case BCH_DISK_ACCOUNTING_nr_inodes: 222 break; 223 case BCH_DISK_ACCOUNTING_persistent_reserved: 224 prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas); 225 break; 226 case BCH_DISK_ACCOUNTING_replicas: 227 bch2_replicas_entry_to_text(out, &k->replicas); 228 break; 229 case BCH_DISK_ACCOUNTING_dev_data_type: 230 prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev); 231 bch2_prt_data_type(out, k->dev_data_type.data_type); 232 break; 233 case BCH_DISK_ACCOUNTING_compression: 234 bch2_prt_compression_type(out, k->compression.type); 235 break; 236 case BCH_DISK_ACCOUNTING_snapshot: 237 prt_printf(out, "id=%u", k->snapshot.id); 238 break; 239 case BCH_DISK_ACCOUNTING_btree: 240 prt_str(out, "btree="); 241 bch2_btree_id_to_text(out, k->btree.id); 242 break; 243 } 244 } 245 246 void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) 247 { 248 struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k); 249 struct disk_accounting_pos acc_k; 250 bpos_to_disk_accounting_pos(&acc_k, k.k->p); 251 252 bch2_accounting_key_to_text(out, &acc_k); 253 254 for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++) 255 prt_printf(out, " %lli", acc.v->d[i]); 256 } 257 258 void bch2_accounting_swab(struct bkey_s k) 259 { 260 for (u64 *p = (u64 *) k.v; 261 p < (u64 *) bkey_val_end(k); 262 p++) 263 *p = swab64(*p); 264 } 265 266 static inline void __accounting_to_replicas(struct bch_replicas_entry_v1 *r, 267 struct disk_accounting_pos *acc) 268 { 269 unsafe_memcpy(r, &acc->replicas, 270 replicas_entry_bytes(&acc->replicas), 271 "variable length struct"); 272 } 273 274 static inline bool accounting_to_replicas(struct bch_replicas_entry_v1 *r, struct bpos p) 275 { 276 struct disk_accounting_pos acc_k; 277 bpos_to_disk_accounting_pos(&acc_k, p); 278 279 switch (acc_k.type) { 280 case BCH_DISK_ACCOUNTING_replicas: 281 __accounting_to_replicas(r, &acc_k); 282 return true; 283 default: 284 return false; 285 } 286 } 287 288 static int bch2_accounting_update_sb_one(struct bch_fs *c, struct bpos p) 289 { 290 struct bch_replicas_padded r; 291 return accounting_to_replicas(&r.e, p) 292 ? bch2_mark_replicas(c, &r.e) 293 : 0; 294 } 295 296 /* 297 * Ensure accounting keys being updated are present in the superblock, when 298 * applicable (i.e. replicas updates) 299 */ 300 int bch2_accounting_update_sb(struct btree_trans *trans) 301 { 302 for (struct jset_entry *i = trans->journal_entries; 303 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); 304 i = vstruct_next(i)) 305 if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) { 306 int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p); 307 if (ret) 308 return ret; 309 } 310 311 return 0; 312 } 313 314 static int __bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a) 315 { 316 struct bch_accounting_mem *acc = &c->accounting; 317 318 /* raced with another insert, already present: */ 319 if (eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 320 accounting_pos_cmp, &a.k->p) < acc->k.nr) 321 return 0; 322 323 struct accounting_mem_entry n = { 324 .pos = a.k->p, 325 .bversion = a.k->bversion, 326 .nr_counters = bch2_accounting_counters(a.k), 327 .v[0] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64), 328 sizeof(u64), GFP_KERNEL), 329 }; 330 331 if (!n.v[0]) 332 goto err; 333 334 if (acc->gc_running) { 335 n.v[1] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64), 336 sizeof(u64), GFP_KERNEL); 337 if (!n.v[1]) 338 goto err; 339 } 340 341 if (darray_push(&acc->k, n)) 342 goto err; 343 344 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 345 accounting_pos_cmp, NULL); 346 347 if (trace_accounting_mem_insert_enabled()) { 348 struct printbuf buf = PRINTBUF; 349 350 bch2_accounting_to_text(&buf, c, a.s_c); 351 trace_accounting_mem_insert(c, buf.buf); 352 printbuf_exit(&buf); 353 } 354 return 0; 355 err: 356 free_percpu(n.v[1]); 357 free_percpu(n.v[0]); 358 return -BCH_ERR_ENOMEM_disk_accounting; 359 } 360 361 int bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a, 362 enum bch_accounting_mode mode) 363 { 364 struct bch_replicas_padded r; 365 366 if (mode != BCH_ACCOUNTING_read && 367 accounting_to_replicas(&r.e, a.k->p) && 368 !bch2_replicas_marked_locked(c, &r.e)) 369 return -BCH_ERR_btree_insert_need_mark_replicas; 370 371 percpu_up_read(&c->mark_lock); 372 percpu_down_write(&c->mark_lock); 373 int ret = __bch2_accounting_mem_insert(c, a); 374 percpu_up_write(&c->mark_lock); 375 percpu_down_read(&c->mark_lock); 376 return ret; 377 } 378 379 static bool accounting_mem_entry_is_zero(struct accounting_mem_entry *e) 380 { 381 for (unsigned i = 0; i < e->nr_counters; i++) 382 if (percpu_u64_get(e->v[0] + i) || 383 (e->v[1] && 384 percpu_u64_get(e->v[1] + i))) 385 return false; 386 return true; 387 } 388 389 void bch2_accounting_mem_gc(struct bch_fs *c) 390 { 391 struct bch_accounting_mem *acc = &c->accounting; 392 393 percpu_down_write(&c->mark_lock); 394 struct accounting_mem_entry *dst = acc->k.data; 395 396 darray_for_each(acc->k, src) { 397 if (accounting_mem_entry_is_zero(src)) { 398 free_percpu(src->v[0]); 399 free_percpu(src->v[1]); 400 } else { 401 *dst++ = *src; 402 } 403 } 404 405 acc->k.nr = dst - acc->k.data; 406 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 407 accounting_pos_cmp, NULL); 408 percpu_up_write(&c->mark_lock); 409 } 410 411 /* 412 * Read out accounting keys for replicas entries, as an array of 413 * bch_replicas_usage entries. 414 * 415 * Note: this may be deprecated/removed at smoe point in the future and replaced 416 * with something more general, it exists to support the ioctl used by the 417 * 'bcachefs fs usage' command. 418 */ 419 int bch2_fs_replicas_usage_read(struct bch_fs *c, darray_char *usage) 420 { 421 struct bch_accounting_mem *acc = &c->accounting; 422 int ret = 0; 423 424 darray_init(usage); 425 426 percpu_down_read(&c->mark_lock); 427 darray_for_each(acc->k, i) { 428 struct { 429 struct bch_replicas_usage r; 430 u8 pad[BCH_BKEY_PTRS_MAX]; 431 } u; 432 433 if (!accounting_to_replicas(&u.r.r, i->pos)) 434 continue; 435 436 u64 sectors; 437 bch2_accounting_mem_read_counters(acc, i - acc->k.data, §ors, 1, false); 438 u.r.sectors = sectors; 439 440 ret = darray_make_room(usage, replicas_usage_bytes(&u.r)); 441 if (ret) 442 break; 443 444 memcpy(&darray_top(*usage), &u.r, replicas_usage_bytes(&u.r)); 445 usage->nr += replicas_usage_bytes(&u.r); 446 } 447 percpu_up_read(&c->mark_lock); 448 449 if (ret) 450 darray_exit(usage); 451 return ret; 452 } 453 454 int bch2_fs_accounting_read(struct bch_fs *c, darray_char *out_buf, unsigned accounting_types_mask) 455 { 456 457 struct bch_accounting_mem *acc = &c->accounting; 458 int ret = 0; 459 460 darray_init(out_buf); 461 462 percpu_down_read(&c->mark_lock); 463 darray_for_each(acc->k, i) { 464 struct disk_accounting_pos a_p; 465 bpos_to_disk_accounting_pos(&a_p, i->pos); 466 467 if (!(accounting_types_mask & BIT(a_p.type))) 468 continue; 469 470 ret = darray_make_room(out_buf, sizeof(struct bkey_i_accounting) + 471 sizeof(u64) * i->nr_counters); 472 if (ret) 473 break; 474 475 struct bkey_i_accounting *a_out = 476 bkey_accounting_init((void *) &darray_top(*out_buf)); 477 set_bkey_val_u64s(&a_out->k, i->nr_counters); 478 a_out->k.p = i->pos; 479 bch2_accounting_mem_read_counters(acc, i - acc->k.data, 480 a_out->v.d, i->nr_counters, false); 481 482 if (!bch2_accounting_key_is_zero(accounting_i_to_s_c(a_out))) 483 out_buf->nr += bkey_bytes(&a_out->k); 484 } 485 486 percpu_up_read(&c->mark_lock); 487 488 if (ret) 489 darray_exit(out_buf); 490 return ret; 491 } 492 493 static void bch2_accounting_free_counters(struct bch_accounting_mem *acc, bool gc) 494 { 495 darray_for_each(acc->k, e) { 496 free_percpu(e->v[gc]); 497 e->v[gc] = NULL; 498 } 499 } 500 501 int bch2_gc_accounting_start(struct bch_fs *c) 502 { 503 struct bch_accounting_mem *acc = &c->accounting; 504 int ret = 0; 505 506 percpu_down_write(&c->mark_lock); 507 darray_for_each(acc->k, e) { 508 e->v[1] = __alloc_percpu_gfp(e->nr_counters * sizeof(u64), 509 sizeof(u64), GFP_KERNEL); 510 if (!e->v[1]) { 511 bch2_accounting_free_counters(acc, true); 512 ret = -BCH_ERR_ENOMEM_disk_accounting; 513 break; 514 } 515 } 516 517 acc->gc_running = !ret; 518 percpu_up_write(&c->mark_lock); 519 520 return ret; 521 } 522 523 int bch2_gc_accounting_done(struct bch_fs *c) 524 { 525 struct bch_accounting_mem *acc = &c->accounting; 526 struct btree_trans *trans = bch2_trans_get(c); 527 struct printbuf buf = PRINTBUF; 528 struct bpos pos = POS_MIN; 529 int ret = 0; 530 531 percpu_down_write(&c->mark_lock); 532 while (1) { 533 unsigned idx = eytzinger0_find_ge(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 534 accounting_pos_cmp, &pos); 535 536 if (idx >= acc->k.nr) 537 break; 538 539 struct accounting_mem_entry *e = acc->k.data + idx; 540 pos = bpos_successor(e->pos); 541 542 struct disk_accounting_pos acc_k; 543 bpos_to_disk_accounting_pos(&acc_k, e->pos); 544 545 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR) 546 continue; 547 548 u64 src_v[BCH_ACCOUNTING_MAX_COUNTERS]; 549 u64 dst_v[BCH_ACCOUNTING_MAX_COUNTERS]; 550 551 unsigned nr = e->nr_counters; 552 bch2_accounting_mem_read_counters(acc, idx, dst_v, nr, false); 553 bch2_accounting_mem_read_counters(acc, idx, src_v, nr, true); 554 555 if (memcmp(dst_v, src_v, nr * sizeof(u64))) { 556 printbuf_reset(&buf); 557 prt_str(&buf, "accounting mismatch for "); 558 bch2_accounting_key_to_text(&buf, &acc_k); 559 560 prt_str(&buf, ": got"); 561 for (unsigned j = 0; j < nr; j++) 562 prt_printf(&buf, " %llu", dst_v[j]); 563 564 prt_str(&buf, " should be"); 565 for (unsigned j = 0; j < nr; j++) 566 prt_printf(&buf, " %llu", src_v[j]); 567 568 for (unsigned j = 0; j < nr; j++) 569 src_v[j] -= dst_v[j]; 570 571 if (fsck_err(trans, accounting_mismatch, "%s", buf.buf)) { 572 percpu_up_write(&c->mark_lock); 573 ret = commit_do(trans, NULL, NULL, 0, 574 bch2_disk_accounting_mod(trans, &acc_k, src_v, nr, false)); 575 percpu_down_write(&c->mark_lock); 576 if (ret) 577 goto err; 578 579 if (!test_bit(BCH_FS_may_go_rw, &c->flags)) { 580 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); 581 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i; 582 583 accounting_key_init(&k_i.k, &acc_k, src_v, nr); 584 bch2_accounting_mem_mod_locked(trans, 585 bkey_i_to_s_c_accounting(&k_i.k), 586 BCH_ACCOUNTING_normal); 587 588 preempt_disable(); 589 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage); 590 struct bch_fs_usage_base *src = &trans->fs_usage_delta; 591 acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64)); 592 preempt_enable(); 593 } 594 } 595 } 596 } 597 err: 598 fsck_err: 599 percpu_up_write(&c->mark_lock); 600 printbuf_exit(&buf); 601 bch2_trans_put(trans); 602 bch_err_fn(c, ret); 603 return ret; 604 } 605 606 static int accounting_read_key(struct btree_trans *trans, struct bkey_s_c k) 607 { 608 struct bch_fs *c = trans->c; 609 610 if (k.k->type != KEY_TYPE_accounting) 611 return 0; 612 613 percpu_down_read(&c->mark_lock); 614 int ret = bch2_accounting_mem_mod_locked(trans, bkey_s_c_to_accounting(k), 615 BCH_ACCOUNTING_read); 616 percpu_up_read(&c->mark_lock); 617 return ret; 618 } 619 620 static int bch2_disk_accounting_validate_late(struct btree_trans *trans, 621 struct disk_accounting_pos acc, 622 u64 *v, unsigned nr) 623 { 624 struct bch_fs *c = trans->c; 625 struct printbuf buf = PRINTBUF; 626 int ret = 0, invalid_dev = -1; 627 628 switch (acc.type) { 629 case BCH_DISK_ACCOUNTING_replicas: { 630 struct bch_replicas_padded r; 631 __accounting_to_replicas(&r.e, &acc); 632 633 for (unsigned i = 0; i < r.e.nr_devs; i++) 634 if (r.e.devs[i] != BCH_SB_MEMBER_INVALID && 635 !bch2_dev_exists(c, r.e.devs[i])) { 636 invalid_dev = r.e.devs[i]; 637 goto invalid_device; 638 } 639 640 /* 641 * All replicas entry checks except for invalid device are done 642 * in bch2_accounting_validate 643 */ 644 BUG_ON(bch2_replicas_entry_validate(&r.e, c, &buf)); 645 646 if (fsck_err_on(!bch2_replicas_marked_locked(c, &r.e), 647 trans, accounting_replicas_not_marked, 648 "accounting not marked in superblock replicas\n%s", 649 (printbuf_reset(&buf), 650 bch2_accounting_key_to_text(&buf, &acc), 651 buf.buf))) { 652 /* 653 * We're not RW yet and still single threaded, dropping 654 * and retaking lock is ok: 655 */ 656 percpu_up_write(&c->mark_lock); 657 ret = bch2_mark_replicas(c, &r.e); 658 if (ret) 659 goto fsck_err; 660 percpu_down_write(&c->mark_lock); 661 } 662 break; 663 } 664 665 case BCH_DISK_ACCOUNTING_dev_data_type: 666 if (!bch2_dev_exists(c, acc.dev_data_type.dev)) { 667 invalid_dev = acc.dev_data_type.dev; 668 goto invalid_device; 669 } 670 break; 671 } 672 673 fsck_err: 674 printbuf_exit(&buf); 675 return ret; 676 invalid_device: 677 if (fsck_err(trans, accounting_to_invalid_device, 678 "accounting entry points to invalid device %i\n%s", 679 invalid_dev, 680 (printbuf_reset(&buf), 681 bch2_accounting_key_to_text(&buf, &acc), 682 buf.buf))) { 683 for (unsigned i = 0; i < nr; i++) 684 v[i] = -v[i]; 685 686 ret = commit_do(trans, NULL, NULL, 0, 687 bch2_disk_accounting_mod(trans, &acc, v, nr, false)) ?: 688 -BCH_ERR_remove_disk_accounting_entry; 689 } else { 690 ret = -BCH_ERR_remove_disk_accounting_entry; 691 } 692 goto fsck_err; 693 } 694 695 /* 696 * At startup time, initialize the in memory accounting from the btree (and 697 * journal) 698 */ 699 int bch2_accounting_read(struct bch_fs *c) 700 { 701 struct bch_accounting_mem *acc = &c->accounting; 702 struct btree_trans *trans = bch2_trans_get(c); 703 struct printbuf buf = PRINTBUF; 704 705 /* 706 * We might run more than once if we rewind to start topology repair or 707 * btree node scan - and those might cause us to get different results, 708 * so we can't just skip if we've already run. 709 * 710 * Instead, zero out any accounting we have: 711 */ 712 percpu_down_write(&c->mark_lock); 713 darray_for_each(acc->k, e) 714 percpu_memset(e->v[0], 0, sizeof(u64) * e->nr_counters); 715 for_each_member_device(c, ca) 716 percpu_memset(ca->usage, 0, sizeof(*ca->usage)); 717 percpu_memset(c->usage, 0, sizeof(*c->usage)); 718 percpu_up_write(&c->mark_lock); 719 720 struct btree_iter iter; 721 bch2_trans_iter_init(trans, &iter, BTREE_ID_accounting, POS_MIN, 722 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots); 723 iter.flags &= ~BTREE_ITER_with_journal; 724 int ret = for_each_btree_key_continue(trans, iter, 725 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({ 726 struct bkey u; 727 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u); 728 729 if (k.k->type != KEY_TYPE_accounting) 730 continue; 731 732 struct disk_accounting_pos acc_k; 733 bpos_to_disk_accounting_pos(&acc_k, k.k->p); 734 735 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR) 736 break; 737 738 if (!bch2_accounting_is_mem(acc_k)) { 739 struct disk_accounting_pos next; 740 memset(&next, 0, sizeof(next)); 741 next.type = acc_k.type + 1; 742 bch2_btree_iter_set_pos(trans, &iter, disk_accounting_pos_to_bpos(&next)); 743 continue; 744 } 745 746 accounting_read_key(trans, k); 747 })); 748 if (ret) 749 goto err; 750 751 struct journal_keys *keys = &c->journal_keys; 752 struct journal_key *dst = keys->data; 753 move_gap(keys, keys->nr); 754 755 darray_for_each(*keys, i) { 756 if (i->k->k.type == KEY_TYPE_accounting) { 757 struct disk_accounting_pos acc_k; 758 bpos_to_disk_accounting_pos(&acc_k, i->k->k.p); 759 760 if (!bch2_accounting_is_mem(acc_k)) 761 continue; 762 763 struct bkey_s_c k = bkey_i_to_s_c(i->k); 764 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, 765 sizeof(acc->k.data[0]), 766 accounting_pos_cmp, &k.k->p); 767 768 bool applied = idx < acc->k.nr && 769 bversion_cmp(acc->k.data[idx].bversion, k.k->bversion) >= 0; 770 771 if (applied) 772 continue; 773 774 if (i + 1 < &darray_top(*keys) && 775 i[1].k->k.type == KEY_TYPE_accounting && 776 !journal_key_cmp(i, i + 1)) { 777 WARN_ON(bversion_cmp(i[0].k->k.bversion, i[1].k->k.bversion) >= 0); 778 779 i[1].journal_seq = i[0].journal_seq; 780 781 bch2_accounting_accumulate(bkey_i_to_accounting(i[1].k), 782 bkey_s_c_to_accounting(k)); 783 continue; 784 } 785 786 ret = accounting_read_key(trans, k); 787 if (ret) 788 goto err; 789 } 790 791 *dst++ = *i; 792 } 793 keys->gap = keys->nr = dst - keys->data; 794 795 percpu_down_write(&c->mark_lock); 796 797 darray_for_each_reverse(acc->k, i) { 798 struct disk_accounting_pos acc_k; 799 bpos_to_disk_accounting_pos(&acc_k, i->pos); 800 801 u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; 802 memset(v, 0, sizeof(v)); 803 804 for (unsigned j = 0; j < i->nr_counters; j++) 805 v[j] = percpu_u64_get(i->v[0] + j); 806 807 /* 808 * If the entry counters are zeroed, it should be treated as 809 * nonexistent - it might point to an invalid device. 810 * 811 * Remove it, so that if it's re-added it gets re-marked in the 812 * superblock: 813 */ 814 ret = bch2_is_zero(v, sizeof(v[0]) * i->nr_counters) 815 ? -BCH_ERR_remove_disk_accounting_entry 816 : bch2_disk_accounting_validate_late(trans, acc_k, v, i->nr_counters); 817 818 if (ret == -BCH_ERR_remove_disk_accounting_entry) { 819 free_percpu(i->v[0]); 820 free_percpu(i->v[1]); 821 darray_remove_item(&acc->k, i); 822 ret = 0; 823 continue; 824 } 825 826 if (ret) 827 goto fsck_err; 828 } 829 830 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 831 accounting_pos_cmp, NULL); 832 833 preempt_disable(); 834 struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage); 835 836 for (unsigned i = 0; i < acc->k.nr; i++) { 837 struct disk_accounting_pos k; 838 bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos); 839 840 u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; 841 bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false); 842 843 switch (k.type) { 844 case BCH_DISK_ACCOUNTING_persistent_reserved: 845 usage->reserved += v[0] * k.persistent_reserved.nr_replicas; 846 break; 847 case BCH_DISK_ACCOUNTING_replicas: 848 fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]); 849 break; 850 case BCH_DISK_ACCOUNTING_dev_data_type: 851 rcu_read_lock(); 852 struct bch_dev *ca = bch2_dev_rcu_noerror(c, k.dev_data_type.dev); 853 if (ca) { 854 struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type]; 855 percpu_u64_set(&d->buckets, v[0]); 856 percpu_u64_set(&d->sectors, v[1]); 857 percpu_u64_set(&d->fragmented, v[2]); 858 859 if (k.dev_data_type.data_type == BCH_DATA_sb || 860 k.dev_data_type.data_type == BCH_DATA_journal) 861 usage->hidden += v[0] * ca->mi.bucket_size; 862 } 863 rcu_read_unlock(); 864 break; 865 } 866 } 867 preempt_enable(); 868 fsck_err: 869 percpu_up_write(&c->mark_lock); 870 err: 871 printbuf_exit(&buf); 872 bch2_trans_put(trans); 873 bch_err_fn(c, ret); 874 return ret; 875 } 876 877 int bch2_dev_usage_remove(struct bch_fs *c, unsigned dev) 878 { 879 return bch2_trans_run(c, 880 bch2_btree_write_buffer_flush_sync(trans) ?: 881 for_each_btree_key_commit(trans, iter, BTREE_ID_accounting, POS_MIN, 882 BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({ 883 struct disk_accounting_pos acc; 884 bpos_to_disk_accounting_pos(&acc, k.k->p); 885 886 acc.type == BCH_DISK_ACCOUNTING_dev_data_type && 887 acc.dev_data_type.dev == dev 888 ? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0) 889 : 0; 890 })) ?: 891 bch2_btree_write_buffer_flush_sync(trans)); 892 } 893 894 int bch2_dev_usage_init(struct bch_dev *ca, bool gc) 895 { 896 struct bch_fs *c = ca->fs; 897 u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 }; 898 899 int ret = bch2_trans_do(c, ({ 900 bch2_disk_accounting_mod2(trans, gc, 901 v, dev_data_type, 902 .dev = ca->dev_idx, 903 .data_type = BCH_DATA_free) ?: 904 (!gc ? bch2_trans_commit(trans, NULL, NULL, 0) : 0); 905 })); 906 bch_err_fn(c, ret); 907 return ret; 908 } 909 910 void bch2_verify_accounting_clean(struct bch_fs *c) 911 { 912 bool mismatch = false; 913 struct bch_fs_usage_base base = {}, base_inmem = {}; 914 915 bch2_trans_run(c, 916 for_each_btree_key(trans, iter, 917 BTREE_ID_accounting, POS_MIN, 918 BTREE_ITER_all_snapshots, k, ({ 919 u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; 920 struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k); 921 unsigned nr = bch2_accounting_counters(k.k); 922 923 struct disk_accounting_pos acc_k; 924 bpos_to_disk_accounting_pos(&acc_k, k.k->p); 925 926 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR) 927 break; 928 929 if (!bch2_accounting_is_mem(acc_k)) { 930 struct disk_accounting_pos next; 931 memset(&next, 0, sizeof(next)); 932 next.type = acc_k.type + 1; 933 bch2_btree_iter_set_pos(trans, &iter, disk_accounting_pos_to_bpos(&next)); 934 continue; 935 } 936 937 bch2_accounting_mem_read(c, k.k->p, v, nr); 938 939 if (memcmp(a.v->d, v, nr * sizeof(u64))) { 940 struct printbuf buf = PRINTBUF; 941 942 bch2_bkey_val_to_text(&buf, c, k); 943 prt_str(&buf, " !="); 944 for (unsigned j = 0; j < nr; j++) 945 prt_printf(&buf, " %llu", v[j]); 946 947 pr_err("%s", buf.buf); 948 printbuf_exit(&buf); 949 mismatch = true; 950 } 951 952 switch (acc_k.type) { 953 case BCH_DISK_ACCOUNTING_persistent_reserved: 954 base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0]; 955 break; 956 case BCH_DISK_ACCOUNTING_replicas: 957 fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]); 958 break; 959 case BCH_DISK_ACCOUNTING_dev_data_type: { 960 rcu_read_lock(); 961 struct bch_dev *ca = bch2_dev_rcu_noerror(c, acc_k.dev_data_type.dev); 962 if (!ca) { 963 rcu_read_unlock(); 964 continue; 965 } 966 967 v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets); 968 v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors); 969 v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented); 970 rcu_read_unlock(); 971 972 if (memcmp(a.v->d, v, 3 * sizeof(u64))) { 973 struct printbuf buf = PRINTBUF; 974 975 bch2_bkey_val_to_text(&buf, c, k); 976 prt_str(&buf, " in mem"); 977 for (unsigned j = 0; j < nr; j++) 978 prt_printf(&buf, " %llu", v[j]); 979 980 pr_err("dev accounting mismatch: %s", buf.buf); 981 printbuf_exit(&buf); 982 mismatch = true; 983 } 984 } 985 } 986 987 0; 988 }))); 989 990 acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64)); 991 992 #define check(x) \ 993 if (base.x != base_inmem.x) { \ 994 pr_err("fs_usage_base.%s mismatch: %llu != %llu", #x, base.x, base_inmem.x); \ 995 mismatch = true; \ 996 } 997 998 //check(hidden); 999 check(btree); 1000 check(data); 1001 check(cached); 1002 check(reserved); 1003 check(nr_inodes); 1004 1005 WARN_ON(mismatch); 1006 } 1007 1008 void bch2_accounting_gc_free(struct bch_fs *c) 1009 { 1010 lockdep_assert_held(&c->mark_lock); 1011 1012 struct bch_accounting_mem *acc = &c->accounting; 1013 1014 bch2_accounting_free_counters(acc, true); 1015 acc->gc_running = false; 1016 } 1017 1018 void bch2_fs_accounting_exit(struct bch_fs *c) 1019 { 1020 struct bch_accounting_mem *acc = &c->accounting; 1021 1022 bch2_accounting_free_counters(acc, false); 1023 darray_exit(&acc->k); 1024 } 1025