1 // SPDX-License-Identifier: GPL-2.0
2
3 #include "bcachefs.h"
4 #include "bcachefs_ioctl.h"
5 #include "btree_cache.h"
6 #include "btree_journal_iter.h"
7 #include "btree_update.h"
8 #include "btree_write_buffer.h"
9 #include "buckets.h"
10 #include "compress.h"
11 #include "disk_accounting.h"
12 #include "error.h"
13 #include "journal_io.h"
14 #include "replicas.h"
15
16 /*
17 * Notes on disk accounting:
18 *
19 * We have two parallel sets of counters to be concerned with, and both must be
20 * kept in sync.
21 *
22 * - Persistent/on disk accounting, stored in the accounting btree and updated
23 * via btree write buffer updates that treat new accounting keys as deltas to
24 * apply to existing values. But reading from a write buffer btree is
25 * expensive, so we also have
26 *
27 * - In memory accounting, where accounting is stored as an array of percpu
28 * counters, indexed by an eytzinger array of disk acounting keys/bpos (which
29 * are the same thing, excepting byte swabbing on big endian).
30 *
31 * Cheap to read, but non persistent.
32 *
33 * Disk accounting updates are generated by transactional triggers; these run as
34 * keys enter and leave the btree, and can compare old and new versions of keys;
35 * the output of these triggers are deltas to the various counters.
36 *
37 * Disk accounting updates are done as btree write buffer updates, where the
38 * counters in the disk accounting key are deltas that will be applied to the
39 * counter in the btree when the key is flushed by the write buffer (or journal
40 * replay).
41 *
42 * To do a disk accounting update:
43 * - initialize a disk_accounting_pos, to specify which counter is being update
44 * - initialize counter deltas, as an array of 1-3 s64s
45 * - call bch2_disk_accounting_mod()
46 *
47 * This queues up the accounting update to be done at transaction commit time.
48 * Underneath, it's a normal btree write buffer update.
49 *
50 * The transaction commit path is responsible for propagating updates to the in
51 * memory counters, with bch2_accounting_mem_mod().
52 *
53 * The commit path also assigns every disk accounting update a unique version
54 * number, based on the journal sequence number and offset within that journal
55 * buffer; this is used by journal replay to determine which updates have been
56 * done.
57 *
58 * The transaction commit path also ensures that replicas entry accounting
59 * updates are properly marked in the superblock (so that we know whether we can
60 * mount without data being unavailable); it will update the superblock if
61 * bch2_accounting_mem_mod() tells it to.
62 */
63
64 static const char * const disk_accounting_type_strs[] = {
65 #define x(t, n, ...) [n] = #t,
66 BCH_DISK_ACCOUNTING_TYPES()
67 #undef x
68 NULL
69 };
70
accounting_key_init(struct bkey_i * k,struct disk_accounting_pos * pos,s64 * d,unsigned nr)71 static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos *pos,
72 s64 *d, unsigned nr)
73 {
74 struct bkey_i_accounting *acc = bkey_accounting_init(k);
75
76 acc->k.p = disk_accounting_pos_to_bpos(pos);
77 set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr);
78
79 memcpy_u64s_small(acc->v.d, d, nr);
80 }
81
82 static int bch2_accounting_update_sb_one(struct bch_fs *, struct bpos);
83
bch2_disk_accounting_mod(struct btree_trans * trans,struct disk_accounting_pos * k,s64 * d,unsigned nr,bool gc)84 int bch2_disk_accounting_mod(struct btree_trans *trans,
85 struct disk_accounting_pos *k,
86 s64 *d, unsigned nr, bool gc)
87 {
88 /* Normalize: */
89 switch (k->type) {
90 case BCH_DISK_ACCOUNTING_replicas:
91 bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp);
92 break;
93 }
94
95 BUG_ON(nr > BCH_ACCOUNTING_MAX_COUNTERS);
96
97 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;
98
99 accounting_key_init(&k_i.k, k, d, nr);
100
101 if (unlikely(gc)) {
102 int ret = bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true);
103 if (ret == -BCH_ERR_btree_insert_need_mark_replicas)
104 ret = drop_locks_do(trans,
105 bch2_accounting_update_sb_one(trans->c, disk_accounting_pos_to_bpos(k))) ?:
106 bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true);
107 return ret;
108 } else {
109 return bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k);
110 }
111 }
112
bch2_mod_dev_cached_sectors(struct btree_trans * trans,unsigned dev,s64 sectors,bool gc)113 int bch2_mod_dev_cached_sectors(struct btree_trans *trans,
114 unsigned dev, s64 sectors,
115 bool gc)
116 {
117 struct disk_accounting_pos acc = {
118 .type = BCH_DISK_ACCOUNTING_replicas,
119 };
120
121 bch2_replicas_entry_cached(&acc.replicas, dev);
122
123 return bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc);
124 }
125
is_zero(char * start,char * end)126 static inline bool is_zero(char *start, char *end)
127 {
128 BUG_ON(start > end);
129
130 for (; start < end; start++)
131 if (*start)
132 return false;
133 return true;
134 }
135
136 #define field_end(p, member) (((void *) (&p.member)) + sizeof(p.member))
137
bch2_accounting_validate(struct bch_fs * c,struct bkey_s_c k,struct bkey_validate_context from)138 int bch2_accounting_validate(struct bch_fs *c, struct bkey_s_c k,
139 struct bkey_validate_context from)
140 {
141 struct disk_accounting_pos acc_k;
142 bpos_to_disk_accounting_pos(&acc_k, k.k->p);
143 void *end = &acc_k + 1;
144 int ret = 0;
145
146 bkey_fsck_err_on((from.flags & BCH_VALIDATE_commit) &&
147 bversion_zero(k.k->bversion),
148 c, accounting_key_version_0,
149 "accounting key with version=0");
150
151 switch (acc_k.type) {
152 case BCH_DISK_ACCOUNTING_nr_inodes:
153 end = field_end(acc_k, nr_inodes);
154 break;
155 case BCH_DISK_ACCOUNTING_persistent_reserved:
156 end = field_end(acc_k, persistent_reserved);
157 break;
158 case BCH_DISK_ACCOUNTING_replicas:
159 bkey_fsck_err_on(!acc_k.replicas.nr_devs,
160 c, accounting_key_replicas_nr_devs_0,
161 "accounting key replicas entry with nr_devs=0");
162
163 bkey_fsck_err_on(acc_k.replicas.nr_required > acc_k.replicas.nr_devs ||
164 (acc_k.replicas.nr_required > 1 &&
165 acc_k.replicas.nr_required == acc_k.replicas.nr_devs),
166 c, accounting_key_replicas_nr_required_bad,
167 "accounting key replicas entry with bad nr_required");
168
169 for (unsigned i = 0; i + 1 < acc_k.replicas.nr_devs; i++)
170 bkey_fsck_err_on(acc_k.replicas.devs[i] >= acc_k.replicas.devs[i + 1],
171 c, accounting_key_replicas_devs_unsorted,
172 "accounting key replicas entry with unsorted devs");
173
174 end = (void *) &acc_k.replicas + replicas_entry_bytes(&acc_k.replicas);
175 break;
176 case BCH_DISK_ACCOUNTING_dev_data_type:
177 end = field_end(acc_k, dev_data_type);
178 break;
179 case BCH_DISK_ACCOUNTING_compression:
180 end = field_end(acc_k, compression);
181 break;
182 case BCH_DISK_ACCOUNTING_snapshot:
183 end = field_end(acc_k, snapshot);
184 break;
185 case BCH_DISK_ACCOUNTING_btree:
186 end = field_end(acc_k, btree);
187 break;
188 case BCH_DISK_ACCOUNTING_rebalance_work:
189 end = field_end(acc_k, rebalance_work);
190 break;
191 }
192
193 bkey_fsck_err_on(!is_zero(end, (void *) (&acc_k + 1)),
194 c, accounting_key_junk_at_end,
195 "junk at end of accounting key");
196 fsck_err:
197 return ret;
198 }
199
bch2_accounting_key_to_text(struct printbuf * out,struct disk_accounting_pos * k)200 void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k)
201 {
202 if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) {
203 prt_printf(out, "unknown type %u", k->type);
204 return;
205 }
206
207 prt_str(out, disk_accounting_type_strs[k->type]);
208 prt_str(out, " ");
209
210 switch (k->type) {
211 case BCH_DISK_ACCOUNTING_nr_inodes:
212 break;
213 case BCH_DISK_ACCOUNTING_persistent_reserved:
214 prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas);
215 break;
216 case BCH_DISK_ACCOUNTING_replicas:
217 bch2_replicas_entry_to_text(out, &k->replicas);
218 break;
219 case BCH_DISK_ACCOUNTING_dev_data_type:
220 prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev);
221 bch2_prt_data_type(out, k->dev_data_type.data_type);
222 break;
223 case BCH_DISK_ACCOUNTING_compression:
224 bch2_prt_compression_type(out, k->compression.type);
225 break;
226 case BCH_DISK_ACCOUNTING_snapshot:
227 prt_printf(out, "id=%u", k->snapshot.id);
228 break;
229 case BCH_DISK_ACCOUNTING_btree:
230 prt_str(out, "btree=");
231 bch2_btree_id_to_text(out, k->btree.id);
232 break;
233 }
234 }
235
bch2_accounting_to_text(struct printbuf * out,struct bch_fs * c,struct bkey_s_c k)236 void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)
237 {
238 struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k);
239 struct disk_accounting_pos acc_k;
240 bpos_to_disk_accounting_pos(&acc_k, k.k->p);
241
242 bch2_accounting_key_to_text(out, &acc_k);
243
244 for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++)
245 prt_printf(out, " %lli", acc.v->d[i]);
246 }
247
bch2_accounting_swab(struct bkey_s k)248 void bch2_accounting_swab(struct bkey_s k)
249 {
250 for (u64 *p = (u64 *) k.v;
251 p < (u64 *) bkey_val_end(k);
252 p++)
253 *p = swab64(*p);
254 }
255
__accounting_to_replicas(struct bch_replicas_entry_v1 * r,struct disk_accounting_pos * acc)256 static inline void __accounting_to_replicas(struct bch_replicas_entry_v1 *r,
257 struct disk_accounting_pos *acc)
258 {
259 unsafe_memcpy(r, &acc->replicas,
260 replicas_entry_bytes(&acc->replicas),
261 "variable length struct");
262 }
263
accounting_to_replicas(struct bch_replicas_entry_v1 * r,struct bpos p)264 static inline bool accounting_to_replicas(struct bch_replicas_entry_v1 *r, struct bpos p)
265 {
266 struct disk_accounting_pos acc_k;
267 bpos_to_disk_accounting_pos(&acc_k, p);
268
269 switch (acc_k.type) {
270 case BCH_DISK_ACCOUNTING_replicas:
271 __accounting_to_replicas(r, &acc_k);
272 return true;
273 default:
274 return false;
275 }
276 }
277
bch2_accounting_update_sb_one(struct bch_fs * c,struct bpos p)278 static int bch2_accounting_update_sb_one(struct bch_fs *c, struct bpos p)
279 {
280 struct bch_replicas_padded r;
281 return accounting_to_replicas(&r.e, p)
282 ? bch2_mark_replicas(c, &r.e)
283 : 0;
284 }
285
286 /*
287 * Ensure accounting keys being updated are present in the superblock, when
288 * applicable (i.e. replicas updates)
289 */
bch2_accounting_update_sb(struct btree_trans * trans)290 int bch2_accounting_update_sb(struct btree_trans *trans)
291 {
292 for (struct jset_entry *i = trans->journal_entries;
293 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
294 i = vstruct_next(i))
295 if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) {
296 int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p);
297 if (ret)
298 return ret;
299 }
300
301 return 0;
302 }
303
__bch2_accounting_mem_insert(struct bch_fs * c,struct bkey_s_c_accounting a)304 static int __bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a)
305 {
306 struct bch_accounting_mem *acc = &c->accounting;
307
308 /* raced with another insert, already present: */
309 if (eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
310 accounting_pos_cmp, &a.k->p) < acc->k.nr)
311 return 0;
312
313 struct accounting_mem_entry n = {
314 .pos = a.k->p,
315 .bversion = a.k->bversion,
316 .nr_counters = bch2_accounting_counters(a.k),
317 .v[0] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64),
318 sizeof(u64), GFP_KERNEL),
319 };
320
321 if (!n.v[0])
322 goto err;
323
324 if (acc->gc_running) {
325 n.v[1] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64),
326 sizeof(u64), GFP_KERNEL);
327 if (!n.v[1])
328 goto err;
329 }
330
331 if (darray_push(&acc->k, n))
332 goto err;
333
334 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
335 accounting_pos_cmp, NULL);
336
337 if (trace_accounting_mem_insert_enabled()) {
338 struct printbuf buf = PRINTBUF;
339
340 bch2_accounting_to_text(&buf, c, a.s_c);
341 trace_accounting_mem_insert(c, buf.buf);
342 printbuf_exit(&buf);
343 }
344 return 0;
345 err:
346 free_percpu(n.v[1]);
347 free_percpu(n.v[0]);
348 return -BCH_ERR_ENOMEM_disk_accounting;
349 }
350
bch2_accounting_mem_insert(struct bch_fs * c,struct bkey_s_c_accounting a,enum bch_accounting_mode mode)351 int bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a,
352 enum bch_accounting_mode mode)
353 {
354 struct bch_replicas_padded r;
355
356 if (mode != BCH_ACCOUNTING_read &&
357 accounting_to_replicas(&r.e, a.k->p) &&
358 !bch2_replicas_marked_locked(c, &r.e))
359 return -BCH_ERR_btree_insert_need_mark_replicas;
360
361 percpu_up_read(&c->mark_lock);
362 percpu_down_write(&c->mark_lock);
363 int ret = __bch2_accounting_mem_insert(c, a);
364 percpu_up_write(&c->mark_lock);
365 percpu_down_read(&c->mark_lock);
366 return ret;
367 }
368
accounting_mem_entry_is_zero(struct accounting_mem_entry * e)369 static bool accounting_mem_entry_is_zero(struct accounting_mem_entry *e)
370 {
371 for (unsigned i = 0; i < e->nr_counters; i++)
372 if (percpu_u64_get(e->v[0] + i) ||
373 (e->v[1] &&
374 percpu_u64_get(e->v[1] + i)))
375 return false;
376 return true;
377 }
378
bch2_accounting_mem_gc(struct bch_fs * c)379 void bch2_accounting_mem_gc(struct bch_fs *c)
380 {
381 struct bch_accounting_mem *acc = &c->accounting;
382
383 percpu_down_write(&c->mark_lock);
384 struct accounting_mem_entry *dst = acc->k.data;
385
386 darray_for_each(acc->k, src) {
387 if (accounting_mem_entry_is_zero(src)) {
388 free_percpu(src->v[0]);
389 free_percpu(src->v[1]);
390 } else {
391 *dst++ = *src;
392 }
393 }
394
395 acc->k.nr = dst - acc->k.data;
396 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
397 accounting_pos_cmp, NULL);
398 percpu_up_write(&c->mark_lock);
399 }
400
401 /*
402 * Read out accounting keys for replicas entries, as an array of
403 * bch_replicas_usage entries.
404 *
405 * Note: this may be deprecated/removed at smoe point in the future and replaced
406 * with something more general, it exists to support the ioctl used by the
407 * 'bcachefs fs usage' command.
408 */
bch2_fs_replicas_usage_read(struct bch_fs * c,darray_char * usage)409 int bch2_fs_replicas_usage_read(struct bch_fs *c, darray_char *usage)
410 {
411 struct bch_accounting_mem *acc = &c->accounting;
412 int ret = 0;
413
414 darray_init(usage);
415
416 percpu_down_read(&c->mark_lock);
417 darray_for_each(acc->k, i) {
418 struct {
419 struct bch_replicas_usage r;
420 u8 pad[BCH_BKEY_PTRS_MAX];
421 } u;
422
423 if (!accounting_to_replicas(&u.r.r, i->pos))
424 continue;
425
426 u64 sectors;
427 bch2_accounting_mem_read_counters(acc, i - acc->k.data, §ors, 1, false);
428 u.r.sectors = sectors;
429
430 ret = darray_make_room(usage, replicas_usage_bytes(&u.r));
431 if (ret)
432 break;
433
434 memcpy(&darray_top(*usage), &u.r, replicas_usage_bytes(&u.r));
435 usage->nr += replicas_usage_bytes(&u.r);
436 }
437 percpu_up_read(&c->mark_lock);
438
439 if (ret)
440 darray_exit(usage);
441 return ret;
442 }
443
bch2_fs_accounting_read(struct bch_fs * c,darray_char * out_buf,unsigned accounting_types_mask)444 int bch2_fs_accounting_read(struct bch_fs *c, darray_char *out_buf, unsigned accounting_types_mask)
445 {
446
447 struct bch_accounting_mem *acc = &c->accounting;
448 int ret = 0;
449
450 darray_init(out_buf);
451
452 percpu_down_read(&c->mark_lock);
453 darray_for_each(acc->k, i) {
454 struct disk_accounting_pos a_p;
455 bpos_to_disk_accounting_pos(&a_p, i->pos);
456
457 if (!(accounting_types_mask & BIT(a_p.type)))
458 continue;
459
460 ret = darray_make_room(out_buf, sizeof(struct bkey_i_accounting) +
461 sizeof(u64) * i->nr_counters);
462 if (ret)
463 break;
464
465 struct bkey_i_accounting *a_out =
466 bkey_accounting_init((void *) &darray_top(*out_buf));
467 set_bkey_val_u64s(&a_out->k, i->nr_counters);
468 a_out->k.p = i->pos;
469 bch2_accounting_mem_read_counters(acc, i - acc->k.data,
470 a_out->v.d, i->nr_counters, false);
471
472 if (!bch2_accounting_key_is_zero(accounting_i_to_s_c(a_out)))
473 out_buf->nr += bkey_bytes(&a_out->k);
474 }
475
476 percpu_up_read(&c->mark_lock);
477
478 if (ret)
479 darray_exit(out_buf);
480 return ret;
481 }
482
bch2_accounting_free_counters(struct bch_accounting_mem * acc,bool gc)483 static void bch2_accounting_free_counters(struct bch_accounting_mem *acc, bool gc)
484 {
485 darray_for_each(acc->k, e) {
486 free_percpu(e->v[gc]);
487 e->v[gc] = NULL;
488 }
489 }
490
bch2_gc_accounting_start(struct bch_fs * c)491 int bch2_gc_accounting_start(struct bch_fs *c)
492 {
493 struct bch_accounting_mem *acc = &c->accounting;
494 int ret = 0;
495
496 percpu_down_write(&c->mark_lock);
497 darray_for_each(acc->k, e) {
498 e->v[1] = __alloc_percpu_gfp(e->nr_counters * sizeof(u64),
499 sizeof(u64), GFP_KERNEL);
500 if (!e->v[1]) {
501 bch2_accounting_free_counters(acc, true);
502 ret = -BCH_ERR_ENOMEM_disk_accounting;
503 break;
504 }
505 }
506
507 acc->gc_running = !ret;
508 percpu_up_write(&c->mark_lock);
509
510 return ret;
511 }
512
bch2_gc_accounting_done(struct bch_fs * c)513 int bch2_gc_accounting_done(struct bch_fs *c)
514 {
515 struct bch_accounting_mem *acc = &c->accounting;
516 struct btree_trans *trans = bch2_trans_get(c);
517 struct printbuf buf = PRINTBUF;
518 struct bpos pos = POS_MIN;
519 int ret = 0;
520
521 percpu_down_write(&c->mark_lock);
522 while (1) {
523 unsigned idx = eytzinger0_find_ge(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
524 accounting_pos_cmp, &pos);
525
526 if (idx >= acc->k.nr)
527 break;
528
529 struct accounting_mem_entry *e = acc->k.data + idx;
530 pos = bpos_successor(e->pos);
531
532 struct disk_accounting_pos acc_k;
533 bpos_to_disk_accounting_pos(&acc_k, e->pos);
534
535 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)
536 continue;
537
538 u64 src_v[BCH_ACCOUNTING_MAX_COUNTERS];
539 u64 dst_v[BCH_ACCOUNTING_MAX_COUNTERS];
540
541 unsigned nr = e->nr_counters;
542 bch2_accounting_mem_read_counters(acc, idx, dst_v, nr, false);
543 bch2_accounting_mem_read_counters(acc, idx, src_v, nr, true);
544
545 if (memcmp(dst_v, src_v, nr * sizeof(u64))) {
546 printbuf_reset(&buf);
547 prt_str(&buf, "accounting mismatch for ");
548 bch2_accounting_key_to_text(&buf, &acc_k);
549
550 prt_str(&buf, ": got");
551 for (unsigned j = 0; j < nr; j++)
552 prt_printf(&buf, " %llu", dst_v[j]);
553
554 prt_str(&buf, " should be");
555 for (unsigned j = 0; j < nr; j++)
556 prt_printf(&buf, " %llu", src_v[j]);
557
558 for (unsigned j = 0; j < nr; j++)
559 src_v[j] -= dst_v[j];
560
561 if (fsck_err(trans, accounting_mismatch, "%s", buf.buf)) {
562 percpu_up_write(&c->mark_lock);
563 ret = commit_do(trans, NULL, NULL, 0,
564 bch2_disk_accounting_mod(trans, &acc_k, src_v, nr, false));
565 percpu_down_write(&c->mark_lock);
566 if (ret)
567 goto err;
568
569 if (!test_bit(BCH_FS_may_go_rw, &c->flags)) {
570 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));
571 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;
572
573 accounting_key_init(&k_i.k, &acc_k, src_v, nr);
574 bch2_accounting_mem_mod_locked(trans,
575 bkey_i_to_s_c_accounting(&k_i.k),
576 BCH_ACCOUNTING_normal);
577
578 preempt_disable();
579 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage);
580 struct bch_fs_usage_base *src = &trans->fs_usage_delta;
581 acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64));
582 preempt_enable();
583 }
584 }
585 }
586 }
587 err:
588 fsck_err:
589 percpu_up_write(&c->mark_lock);
590 printbuf_exit(&buf);
591 bch2_trans_put(trans);
592 bch_err_fn(c, ret);
593 return ret;
594 }
595
accounting_read_key(struct btree_trans * trans,struct bkey_s_c k)596 static int accounting_read_key(struct btree_trans *trans, struct bkey_s_c k)
597 {
598 struct bch_fs *c = trans->c;
599
600 if (k.k->type != KEY_TYPE_accounting)
601 return 0;
602
603 percpu_down_read(&c->mark_lock);
604 int ret = bch2_accounting_mem_mod_locked(trans, bkey_s_c_to_accounting(k),
605 BCH_ACCOUNTING_read);
606 percpu_up_read(&c->mark_lock);
607 return ret;
608 }
609
bch2_disk_accounting_validate_late(struct btree_trans * trans,struct disk_accounting_pos acc,u64 * v,unsigned nr)610 static int bch2_disk_accounting_validate_late(struct btree_trans *trans,
611 struct disk_accounting_pos acc,
612 u64 *v, unsigned nr)
613 {
614 struct bch_fs *c = trans->c;
615 struct printbuf buf = PRINTBUF;
616 int ret = 0, invalid_dev = -1;
617
618 switch (acc.type) {
619 case BCH_DISK_ACCOUNTING_replicas: {
620 struct bch_replicas_padded r;
621 __accounting_to_replicas(&r.e, &acc);
622
623 for (unsigned i = 0; i < r.e.nr_devs; i++)
624 if (r.e.devs[i] != BCH_SB_MEMBER_INVALID &&
625 !bch2_dev_exists(c, r.e.devs[i])) {
626 invalid_dev = r.e.devs[i];
627 goto invalid_device;
628 }
629
630 /*
631 * All replicas entry checks except for invalid device are done
632 * in bch2_accounting_validate
633 */
634 BUG_ON(bch2_replicas_entry_validate(&r.e, c, &buf));
635
636 if (fsck_err_on(!bch2_replicas_marked_locked(c, &r.e),
637 trans, accounting_replicas_not_marked,
638 "accounting not marked in superblock replicas\n %s",
639 (printbuf_reset(&buf),
640 bch2_accounting_key_to_text(&buf, &acc),
641 buf.buf))) {
642 /*
643 * We're not RW yet and still single threaded, dropping
644 * and retaking lock is ok:
645 */
646 percpu_up_write(&c->mark_lock);
647 ret = bch2_mark_replicas(c, &r.e);
648 if (ret)
649 goto fsck_err;
650 percpu_down_write(&c->mark_lock);
651 }
652 break;
653 }
654
655 case BCH_DISK_ACCOUNTING_dev_data_type:
656 if (!bch2_dev_exists(c, acc.dev_data_type.dev)) {
657 invalid_dev = acc.dev_data_type.dev;
658 goto invalid_device;
659 }
660 break;
661 }
662
663 fsck_err:
664 printbuf_exit(&buf);
665 return ret;
666 invalid_device:
667 if (fsck_err(trans, accounting_to_invalid_device,
668 "accounting entry points to invalid device %i\n %s",
669 invalid_dev,
670 (printbuf_reset(&buf),
671 bch2_accounting_key_to_text(&buf, &acc),
672 buf.buf))) {
673 for (unsigned i = 0; i < nr; i++)
674 v[i] = -v[i];
675
676 ret = commit_do(trans, NULL, NULL, 0,
677 bch2_disk_accounting_mod(trans, &acc, v, nr, false)) ?:
678 -BCH_ERR_remove_disk_accounting_entry;
679 } else {
680 ret = -BCH_ERR_remove_disk_accounting_entry;
681 }
682 goto fsck_err;
683 }
684
685 /*
686 * At startup time, initialize the in memory accounting from the btree (and
687 * journal)
688 */
bch2_accounting_read(struct bch_fs * c)689 int bch2_accounting_read(struct bch_fs *c)
690 {
691 struct bch_accounting_mem *acc = &c->accounting;
692 struct btree_trans *trans = bch2_trans_get(c);
693 struct printbuf buf = PRINTBUF;
694
695 /*
696 * We might run more than once if we rewind to start topology repair or
697 * btree node scan - and those might cause us to get different results,
698 * so we can't just skip if we've already run.
699 *
700 * Instead, zero out any accounting we have:
701 */
702 percpu_down_write(&c->mark_lock);
703 darray_for_each(acc->k, e)
704 percpu_memset(e->v[0], 0, sizeof(u64) * e->nr_counters);
705 for_each_member_device(c, ca)
706 percpu_memset(ca->usage, 0, sizeof(*ca->usage));
707 percpu_memset(c->usage, 0, sizeof(*c->usage));
708 percpu_up_write(&c->mark_lock);
709
710 struct btree_iter iter;
711 bch2_trans_iter_init(trans, &iter, BTREE_ID_accounting, POS_MIN,
712 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots);
713 iter.flags &= ~BTREE_ITER_with_journal;
714 int ret = for_each_btree_key_continue(trans, iter,
715 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({
716 struct bkey u;
717 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u);
718
719 if (k.k->type != KEY_TYPE_accounting)
720 continue;
721
722 struct disk_accounting_pos acc_k;
723 bpos_to_disk_accounting_pos(&acc_k, k.k->p);
724
725 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)
726 break;
727
728 if (!bch2_accounting_is_mem(acc_k)) {
729 struct disk_accounting_pos next = { .type = acc_k.type + 1 };
730 bch2_btree_iter_set_pos(&iter, disk_accounting_pos_to_bpos(&next));
731 continue;
732 }
733
734 accounting_read_key(trans, k);
735 }));
736 if (ret)
737 goto err;
738
739 struct journal_keys *keys = &c->journal_keys;
740 struct journal_key *dst = keys->data;
741 move_gap(keys, keys->nr);
742
743 darray_for_each(*keys, i) {
744 if (i->k->k.type == KEY_TYPE_accounting) {
745 struct disk_accounting_pos acc_k;
746 bpos_to_disk_accounting_pos(&acc_k, i->k->k.p);
747
748 if (!bch2_accounting_is_mem(acc_k))
749 continue;
750
751 struct bkey_s_c k = bkey_i_to_s_c(i->k);
752 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr,
753 sizeof(acc->k.data[0]),
754 accounting_pos_cmp, &k.k->p);
755
756 bool applied = idx < acc->k.nr &&
757 bversion_cmp(acc->k.data[idx].bversion, k.k->bversion) >= 0;
758
759 if (applied)
760 continue;
761
762 if (i + 1 < &darray_top(*keys) &&
763 i[1].k->k.type == KEY_TYPE_accounting &&
764 !journal_key_cmp(i, i + 1)) {
765 WARN_ON(bversion_cmp(i[0].k->k.bversion, i[1].k->k.bversion) >= 0);
766
767 i[1].journal_seq = i[0].journal_seq;
768
769 bch2_accounting_accumulate(bkey_i_to_accounting(i[1].k),
770 bkey_s_c_to_accounting(k));
771 continue;
772 }
773
774 ret = accounting_read_key(trans, k);
775 if (ret)
776 goto err;
777 }
778
779 *dst++ = *i;
780 }
781 keys->gap = keys->nr = dst - keys->data;
782
783 percpu_down_write(&c->mark_lock);
784
785 darray_for_each_reverse(acc->k, i) {
786 struct disk_accounting_pos acc_k;
787 bpos_to_disk_accounting_pos(&acc_k, i->pos);
788
789 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
790 memset(v, 0, sizeof(v));
791
792 for (unsigned j = 0; j < i->nr_counters; j++)
793 v[j] = percpu_u64_get(i->v[0] + j);
794
795 /*
796 * If the entry counters are zeroed, it should be treated as
797 * nonexistent - it might point to an invalid device.
798 *
799 * Remove it, so that if it's re-added it gets re-marked in the
800 * superblock:
801 */
802 ret = bch2_is_zero(v, sizeof(v[0]) * i->nr_counters)
803 ? -BCH_ERR_remove_disk_accounting_entry
804 : bch2_disk_accounting_validate_late(trans, acc_k, v, i->nr_counters);
805
806 if (ret == -BCH_ERR_remove_disk_accounting_entry) {
807 free_percpu(i->v[0]);
808 free_percpu(i->v[1]);
809 darray_remove_item(&acc->k, i);
810 ret = 0;
811 continue;
812 }
813
814 if (ret)
815 goto fsck_err;
816 }
817
818 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
819 accounting_pos_cmp, NULL);
820
821 preempt_disable();
822 struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage);
823
824 for (unsigned i = 0; i < acc->k.nr; i++) {
825 struct disk_accounting_pos k;
826 bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos);
827
828 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
829 bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false);
830
831 switch (k.type) {
832 case BCH_DISK_ACCOUNTING_persistent_reserved:
833 usage->reserved += v[0] * k.persistent_reserved.nr_replicas;
834 break;
835 case BCH_DISK_ACCOUNTING_replicas:
836 fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]);
837 break;
838 case BCH_DISK_ACCOUNTING_dev_data_type:
839 rcu_read_lock();
840 struct bch_dev *ca = bch2_dev_rcu_noerror(c, k.dev_data_type.dev);
841 if (ca) {
842 struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type];
843 percpu_u64_set(&d->buckets, v[0]);
844 percpu_u64_set(&d->sectors, v[1]);
845 percpu_u64_set(&d->fragmented, v[2]);
846
847 if (k.dev_data_type.data_type == BCH_DATA_sb ||
848 k.dev_data_type.data_type == BCH_DATA_journal)
849 usage->hidden += v[0] * ca->mi.bucket_size;
850 }
851 rcu_read_unlock();
852 break;
853 }
854 }
855 preempt_enable();
856 fsck_err:
857 percpu_up_write(&c->mark_lock);
858 err:
859 printbuf_exit(&buf);
860 bch2_trans_put(trans);
861 bch_err_fn(c, ret);
862 return ret;
863 }
864
bch2_dev_usage_remove(struct bch_fs * c,unsigned dev)865 int bch2_dev_usage_remove(struct bch_fs *c, unsigned dev)
866 {
867 return bch2_trans_run(c,
868 bch2_btree_write_buffer_flush_sync(trans) ?:
869 for_each_btree_key_commit(trans, iter, BTREE_ID_accounting, POS_MIN,
870 BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({
871 struct disk_accounting_pos acc;
872 bpos_to_disk_accounting_pos(&acc, k.k->p);
873
874 acc.type == BCH_DISK_ACCOUNTING_dev_data_type &&
875 acc.dev_data_type.dev == dev
876 ? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0)
877 : 0;
878 })) ?:
879 bch2_btree_write_buffer_flush_sync(trans));
880 }
881
bch2_dev_usage_init(struct bch_dev * ca,bool gc)882 int bch2_dev_usage_init(struct bch_dev *ca, bool gc)
883 {
884 struct bch_fs *c = ca->fs;
885 struct disk_accounting_pos acc = {
886 .type = BCH_DISK_ACCOUNTING_dev_data_type,
887 .dev_data_type.dev = ca->dev_idx,
888 .dev_data_type.data_type = BCH_DATA_free,
889 };
890 u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 };
891
892 int ret = bch2_trans_do(c, ({
893 bch2_disk_accounting_mod(trans, &acc, v, ARRAY_SIZE(v), gc) ?:
894 (!gc ? bch2_trans_commit(trans, NULL, NULL, 0) : 0);
895 }));
896 bch_err_fn(c, ret);
897 return ret;
898 }
899
bch2_verify_accounting_clean(struct bch_fs * c)900 void bch2_verify_accounting_clean(struct bch_fs *c)
901 {
902 bool mismatch = false;
903 struct bch_fs_usage_base base = {}, base_inmem = {};
904
905 bch2_trans_run(c,
906 for_each_btree_key(trans, iter,
907 BTREE_ID_accounting, POS_MIN,
908 BTREE_ITER_all_snapshots, k, ({
909 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
910 struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k);
911 unsigned nr = bch2_accounting_counters(k.k);
912
913 struct disk_accounting_pos acc_k;
914 bpos_to_disk_accounting_pos(&acc_k, k.k->p);
915
916 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)
917 break;
918
919 if (!bch2_accounting_is_mem(acc_k)) {
920 struct disk_accounting_pos next = { .type = acc_k.type + 1 };
921 bch2_btree_iter_set_pos(&iter, disk_accounting_pos_to_bpos(&next));
922 continue;
923 }
924
925 bch2_accounting_mem_read(c, k.k->p, v, nr);
926
927 if (memcmp(a.v->d, v, nr * sizeof(u64))) {
928 struct printbuf buf = PRINTBUF;
929
930 bch2_bkey_val_to_text(&buf, c, k);
931 prt_str(&buf, " !=");
932 for (unsigned j = 0; j < nr; j++)
933 prt_printf(&buf, " %llu", v[j]);
934
935 pr_err("%s", buf.buf);
936 printbuf_exit(&buf);
937 mismatch = true;
938 }
939
940 switch (acc_k.type) {
941 case BCH_DISK_ACCOUNTING_persistent_reserved:
942 base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0];
943 break;
944 case BCH_DISK_ACCOUNTING_replicas:
945 fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]);
946 break;
947 case BCH_DISK_ACCOUNTING_dev_data_type: {
948 rcu_read_lock();
949 struct bch_dev *ca = bch2_dev_rcu_noerror(c, acc_k.dev_data_type.dev);
950 if (!ca) {
951 rcu_read_unlock();
952 continue;
953 }
954
955 v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets);
956 v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors);
957 v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented);
958 rcu_read_unlock();
959
960 if (memcmp(a.v->d, v, 3 * sizeof(u64))) {
961 struct printbuf buf = PRINTBUF;
962
963 bch2_bkey_val_to_text(&buf, c, k);
964 prt_str(&buf, " in mem");
965 for (unsigned j = 0; j < nr; j++)
966 prt_printf(&buf, " %llu", v[j]);
967
968 pr_err("dev accounting mismatch: %s", buf.buf);
969 printbuf_exit(&buf);
970 mismatch = true;
971 }
972 }
973 }
974
975 0;
976 })));
977
978 acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64));
979
980 #define check(x) \
981 if (base.x != base_inmem.x) { \
982 pr_err("fs_usage_base.%s mismatch: %llu != %llu", #x, base.x, base_inmem.x); \
983 mismatch = true; \
984 }
985
986 //check(hidden);
987 check(btree);
988 check(data);
989 check(cached);
990 check(reserved);
991 check(nr_inodes);
992
993 WARN_ON(mismatch);
994 }
995
bch2_accounting_gc_free(struct bch_fs * c)996 void bch2_accounting_gc_free(struct bch_fs *c)
997 {
998 lockdep_assert_held(&c->mark_lock);
999
1000 struct bch_accounting_mem *acc = &c->accounting;
1001
1002 bch2_accounting_free_counters(acc, true);
1003 acc->gc_running = false;
1004 }
1005
bch2_fs_accounting_exit(struct bch_fs * c)1006 void bch2_fs_accounting_exit(struct bch_fs *c)
1007 {
1008 struct bch_accounting_mem *acc = &c->accounting;
1009
1010 bch2_accounting_free_counters(acc, false);
1011 darray_exit(&acc->k);
1012 }
1013