1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _BCACHEFS_DISK_ACCOUNTING_H
3 #define _BCACHEFS_DISK_ACCOUNTING_H
4
5 #include "btree_update.h"
6 #include "eytzinger.h"
7 #include "sb-members.h"
8
bch2_u64s_neg(u64 * v,unsigned nr)9 static inline void bch2_u64s_neg(u64 *v, unsigned nr)
10 {
11 for (unsigned i = 0; i < nr; i++)
12 v[i] = -v[i];
13 }
14
bch2_accounting_counters(const struct bkey * k)15 static inline unsigned bch2_accounting_counters(const struct bkey *k)
16 {
17 return bkey_val_u64s(k) - offsetof(struct bch_accounting, d) / sizeof(u64);
18 }
19
bch2_accounting_neg(struct bkey_s_accounting a)20 static inline void bch2_accounting_neg(struct bkey_s_accounting a)
21 {
22 bch2_u64s_neg(a.v->d, bch2_accounting_counters(a.k));
23 }
24
bch2_accounting_key_is_zero(struct bkey_s_c_accounting a)25 static inline bool bch2_accounting_key_is_zero(struct bkey_s_c_accounting a)
26 {
27 for (unsigned i = 0; i < bch2_accounting_counters(a.k); i++)
28 if (a.v->d[i])
29 return false;
30 return true;
31 }
32
bch2_accounting_accumulate(struct bkey_i_accounting * dst,struct bkey_s_c_accounting src)33 static inline void bch2_accounting_accumulate(struct bkey_i_accounting *dst,
34 struct bkey_s_c_accounting src)
35 {
36 for (unsigned i = 0;
37 i < min(bch2_accounting_counters(&dst->k),
38 bch2_accounting_counters(src.k));
39 i++)
40 dst->v.d[i] += src.v->d[i];
41
42 if (bversion_cmp(dst->k.bversion, src.k->bversion) < 0)
43 dst->k.bversion = src.k->bversion;
44 }
45
fs_usage_data_type_to_base(struct bch_fs_usage_base * fs_usage,enum bch_data_type data_type,s64 sectors)46 static inline void fs_usage_data_type_to_base(struct bch_fs_usage_base *fs_usage,
47 enum bch_data_type data_type,
48 s64 sectors)
49 {
50 switch (data_type) {
51 case BCH_DATA_btree:
52 fs_usage->btree += sectors;
53 break;
54 case BCH_DATA_user:
55 case BCH_DATA_parity:
56 fs_usage->data += sectors;
57 break;
58 case BCH_DATA_cached:
59 fs_usage->cached += sectors;
60 break;
61 default:
62 break;
63 }
64 }
65
bpos_to_disk_accounting_pos(struct disk_accounting_pos * acc,struct bpos p)66 static inline void bpos_to_disk_accounting_pos(struct disk_accounting_pos *acc, struct bpos p)
67 {
68 BUILD_BUG_ON(sizeof(*acc) != sizeof(p));
69
70 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
71 acc->_pad = p;
72 #else
73 memcpy_swab(acc, &p, sizeof(p));
74 #endif
75 }
76
disk_accounting_pos_to_bpos(struct disk_accounting_pos * acc)77 static inline struct bpos disk_accounting_pos_to_bpos(struct disk_accounting_pos *acc)
78 {
79 struct bpos p;
80 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
81 p = acc->_pad;
82 #else
83 memcpy_swab(&p, acc, sizeof(p));
84 #endif
85 return p;
86 }
87
88 int bch2_disk_accounting_mod(struct btree_trans *, struct disk_accounting_pos *,
89 s64 *, unsigned, bool);
90
91 #define disk_accounting_key_init(_k, _type, ...) \
92 do { \
93 memset(&(_k), 0, sizeof(_k)); \
94 (_k).type = BCH_DISK_ACCOUNTING_##_type; \
95 (_k)._type = (struct bch_acct_##_type) { __VA_ARGS__ }; \
96 } while (0)
97
98 #define bch2_disk_accounting_mod2_nr(_trans, _gc, _v, _nr, ...) \
99 ({ \
100 struct disk_accounting_pos pos; \
101 disk_accounting_key_init(pos, __VA_ARGS__); \
102 bch2_disk_accounting_mod(trans, &pos, _v, _nr, _gc); \
103 })
104
105 #define bch2_disk_accounting_mod2(_trans, _gc, _v, ...) \
106 bch2_disk_accounting_mod2_nr(_trans, _gc, _v, ARRAY_SIZE(_v), __VA_ARGS__)
107
108 int bch2_mod_dev_cached_sectors(struct btree_trans *, unsigned, s64, bool);
109
110 int bch2_accounting_validate(struct bch_fs *, struct bkey_s_c,
111 struct bkey_validate_context);
112 void bch2_accounting_key_to_text(struct printbuf *, struct disk_accounting_pos *);
113 void bch2_accounting_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c);
114 void bch2_accounting_swab(struct bkey_s);
115
116 #define bch2_bkey_ops_accounting ((struct bkey_ops) { \
117 .key_validate = bch2_accounting_validate, \
118 .val_to_text = bch2_accounting_to_text, \
119 .swab = bch2_accounting_swab, \
120 .min_val_size = 8, \
121 })
122
123 int bch2_accounting_update_sb(struct btree_trans *);
124
accounting_pos_cmp(const void * _l,const void * _r)125 static inline int accounting_pos_cmp(const void *_l, const void *_r)
126 {
127 const struct bpos *l = _l, *r = _r;
128
129 return bpos_cmp(*l, *r);
130 }
131
132 enum bch_accounting_mode {
133 BCH_ACCOUNTING_normal,
134 BCH_ACCOUNTING_gc,
135 BCH_ACCOUNTING_read,
136 };
137
138 int bch2_accounting_mem_insert(struct bch_fs *, struct bkey_s_c_accounting, enum bch_accounting_mode);
139 int bch2_accounting_mem_insert_locked(struct bch_fs *, struct bkey_s_c_accounting, enum bch_accounting_mode);
140 void bch2_accounting_mem_gc(struct bch_fs *);
141
bch2_accounting_is_mem(struct disk_accounting_pos * acc)142 static inline bool bch2_accounting_is_mem(struct disk_accounting_pos *acc)
143 {
144 return acc->type < BCH_DISK_ACCOUNTING_TYPE_NR &&
145 acc->type != BCH_DISK_ACCOUNTING_inum;
146 }
147
148 /*
149 * Update in memory counters so they match the btree update we're doing; called
150 * from transaction commit path
151 */
bch2_accounting_mem_mod_locked(struct btree_trans * trans,struct bkey_s_c_accounting a,enum bch_accounting_mode mode,bool write_locked)152 static inline int bch2_accounting_mem_mod_locked(struct btree_trans *trans,
153 struct bkey_s_c_accounting a,
154 enum bch_accounting_mode mode,
155 bool write_locked)
156 {
157 struct bch_fs *c = trans->c;
158 struct bch_accounting_mem *acc = &c->accounting;
159 struct disk_accounting_pos acc_k;
160 bpos_to_disk_accounting_pos(&acc_k, a.k->p);
161 bool gc = mode == BCH_ACCOUNTING_gc;
162
163 if (gc && !acc->gc_running)
164 return 0;
165
166 if (!bch2_accounting_is_mem(&acc_k))
167 return 0;
168
169 if (mode == BCH_ACCOUNTING_normal) {
170 switch (acc_k.type) {
171 case BCH_DISK_ACCOUNTING_persistent_reserved:
172 trans->fs_usage_delta.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0];
173 break;
174 case BCH_DISK_ACCOUNTING_replicas:
175 fs_usage_data_type_to_base(&trans->fs_usage_delta, acc_k.replicas.data_type, a.v->d[0]);
176 break;
177 case BCH_DISK_ACCOUNTING_dev_data_type: {
178 guard(rcu)();
179 struct bch_dev *ca = bch2_dev_rcu_noerror(c, acc_k.dev_data_type.dev);
180 if (ca) {
181 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].buckets, a.v->d[0]);
182 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].sectors, a.v->d[1]);
183 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].fragmented, a.v->d[2]);
184 }
185 break;
186 }
187 }
188 }
189
190 unsigned idx;
191
192 while ((idx = eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
193 accounting_pos_cmp, &a.k->p)) >= acc->k.nr) {
194 int ret = 0;
195 if (unlikely(write_locked))
196 ret = bch2_accounting_mem_insert_locked(c, a, mode);
197 else
198 ret = bch2_accounting_mem_insert(c, a, mode);
199 if (ret)
200 return ret;
201 }
202
203 struct accounting_mem_entry *e = &acc->k.data[idx];
204
205 EBUG_ON(bch2_accounting_counters(a.k) != e->nr_counters);
206
207 for (unsigned i = 0; i < bch2_accounting_counters(a.k); i++)
208 this_cpu_add(e->v[gc][i], a.v->d[i]);
209 return 0;
210 }
211
bch2_accounting_mem_add(struct btree_trans * trans,struct bkey_s_c_accounting a,bool gc)212 static inline int bch2_accounting_mem_add(struct btree_trans *trans, struct bkey_s_c_accounting a, bool gc)
213 {
214 percpu_down_read(&trans->c->mark_lock);
215 int ret = bch2_accounting_mem_mod_locked(trans, a, gc ? BCH_ACCOUNTING_gc : BCH_ACCOUNTING_normal, false);
216 percpu_up_read(&trans->c->mark_lock);
217 return ret;
218 }
219
bch2_accounting_mem_read_counters(struct bch_accounting_mem * acc,unsigned idx,u64 * v,unsigned nr,bool gc)220 static inline void bch2_accounting_mem_read_counters(struct bch_accounting_mem *acc,
221 unsigned idx, u64 *v, unsigned nr, bool gc)
222 {
223 memset(v, 0, sizeof(*v) * nr);
224
225 if (unlikely(idx >= acc->k.nr))
226 return;
227
228 struct accounting_mem_entry *e = &acc->k.data[idx];
229
230 nr = min_t(unsigned, nr, e->nr_counters);
231
232 for (unsigned i = 0; i < nr; i++)
233 v[i] = percpu_u64_get(e->v[gc] + i);
234 }
235
bch2_accounting_mem_read(struct bch_fs * c,struct bpos p,u64 * v,unsigned nr)236 static inline void bch2_accounting_mem_read(struct bch_fs *c, struct bpos p,
237 u64 *v, unsigned nr)
238 {
239 percpu_down_read(&c->mark_lock);
240 struct bch_accounting_mem *acc = &c->accounting;
241 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
242 accounting_pos_cmp, &p);
243
244 bch2_accounting_mem_read_counters(acc, idx, v, nr, false);
245 percpu_up_read(&c->mark_lock);
246 }
247
journal_pos_to_bversion(struct journal_res * res,unsigned offset)248 static inline struct bversion journal_pos_to_bversion(struct journal_res *res, unsigned offset)
249 {
250 EBUG_ON(!res->ref);
251
252 return (struct bversion) {
253 .hi = res->seq >> 32,
254 .lo = (res->seq << 32) | (res->offset + offset),
255 };
256 }
257
bch2_accounting_trans_commit_hook(struct btree_trans * trans,struct bkey_i_accounting * a,unsigned commit_flags)258 static inline int bch2_accounting_trans_commit_hook(struct btree_trans *trans,
259 struct bkey_i_accounting *a,
260 unsigned commit_flags)
261 {
262 u64 *base = (u64 *) btree_trans_subbuf_base(trans, &trans->accounting);
263 a->k.bversion = journal_pos_to_bversion(&trans->journal_res, (u64 *) a - base);
264
265 EBUG_ON(bversion_zero(a->k.bversion));
266
267 return likely(!(commit_flags & BCH_TRANS_COMMIT_skip_accounting_apply))
268 ? bch2_accounting_mem_mod_locked(trans, accounting_i_to_s_c(a), BCH_ACCOUNTING_normal, false)
269 : 0;
270 }
271
bch2_accounting_trans_commit_revert(struct btree_trans * trans,struct bkey_i_accounting * a_i,unsigned commit_flags)272 static inline void bch2_accounting_trans_commit_revert(struct btree_trans *trans,
273 struct bkey_i_accounting *a_i,
274 unsigned commit_flags)
275 {
276 if (likely(!(commit_flags & BCH_TRANS_COMMIT_skip_accounting_apply))) {
277 struct bkey_s_accounting a = accounting_i_to_s(a_i);
278
279 bch2_accounting_neg(a);
280 bch2_accounting_mem_mod_locked(trans, a.c, BCH_ACCOUNTING_normal, false);
281 bch2_accounting_neg(a);
282 }
283 }
284
285 int bch2_fs_replicas_usage_read(struct bch_fs *, darray_char *);
286 int bch2_fs_accounting_read(struct bch_fs *, darray_char *, unsigned);
287
288 int bch2_gc_accounting_start(struct bch_fs *);
289 int bch2_gc_accounting_done(struct bch_fs *);
290
291 int bch2_accounting_read(struct bch_fs *);
292
293 int bch2_dev_usage_remove(struct bch_fs *, unsigned);
294 int bch2_dev_usage_init(struct bch_dev *, bool);
295
296 void bch2_verify_accounting_clean(struct bch_fs *c);
297
298 void bch2_accounting_gc_free(struct bch_fs *);
299 void bch2_fs_accounting_exit(struct bch_fs *);
300
301 #endif /* _BCACHEFS_DISK_ACCOUNTING_H */
302